14208558196

Committed 02 Apr 2025 12:10AM UTC coverage: 92.133% (+0.06%) from 92.078%

Build # 14208558196

Build Type

push

github

Committed by

web-flow

Commit Message

2025 03 update (#83)

Go 1.24
Node 23.10
Svelte 5.25.3

Run Details

119 of 159 new or added lines in 18 files covered. (74.84%)

3 existing lines in 3 files now uncovered.

5352 of 5809 relevant lines covered (92.13%)

1.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.95

/pkg/sc/script_def_loader.go

package sc

import (
        "encoding/json"
        "fmt"
        "regexp"
        "strings"
)

type ScriptInitProblemType int

const ScriptInitNoProblem ScriptInitProblemType = 0
const ScriptInitUrlProblem ScriptInitProblemType = 1
const ScriptInitContentProblem ScriptInitProblemType = 2
const ScriptInitConnectivityProblem ScriptInitProblemType = 3

func getFileType(url string) (ScriptType, error) {
        if strings.HasSuffix(url, ".json") {
                return ScriptJson, nil
        } else if strings.HasSuffix(url, ".yaml") {
                return ScriptYaml, nil
        }
        return ScriptUnknown, fmt.Errorf("cannot detect json/yaml file type: %s", url)
}

func NewScriptFromFileBytes(
        caPath string,
        privateKeys map[string]string,
        scriptUrl string,
        jsonOrYamlBytesScript []byte,
        scriptParamsUrl string,
        jsonOrYamlBytesParams []byte,
        customProcessorDefFactoryInstance CustomProcessorDefFactory,
        customProcessorsSettings map[string]json.RawMessage) (*ScriptDef, ScriptInitProblemType, error) {
        // Make sure parameters are in canonical format: {param_name|param_type}
        jsonOrYamlScriptString := string(jsonOrYamlBytesScript)

        // Default param type is string: {param} -> {param|string}
        re := regexp.MustCompile("{[ ]*([a-zA-Z0-9_]+)[ ]*}")
        jsonOrYamlScriptString = re.ReplaceAllString(jsonOrYamlScriptString, "{$1|string}")

        // Remove spaces: {  param_name | param_type } -> {param_name|param_type}
        re = regexp.MustCompile(`{[ ]*([a-zA-Z0-9_]+)[ ]*\|[ ]*(string|number|bool|stringlist)[ ]*}`)
        jsonOrYamlScriptString = re.ReplaceAllString(jsonOrYamlScriptString, "{$1|$2}")

        // Verify that number/bool must be like "{param_name|number}", no extra characters between double quotes and curly braces
        // This is a big limitation of the existing parameter implementation.
        // Those double quotes must be there in order to keep JSON well-formed.
        // Because of this, using number/bool parameters in strings is not allowed, use string paramater like "10" and "false" in those cases.
        re = regexp.MustCompile(`([^"]{[a-zA-Z0-9_]+\|(number|bool)})|({[a-zA-Z0-9_]+\|(number|bool)}[^"])`)
        invalidParamRefs := re.FindAllString(jsonOrYamlScriptString, -1)
        if len(invalidParamRefs) > 0 {
                return nil, ScriptInitUrlProblem, fmt.Errorf("cannot parse number/bool script parameter references in [%s], the following parameter references should not have extra characters between curly braces and double quotes: [%s]", scriptUrl, strings.Join(invalidParamRefs, ","))
        }

        // Apply template params here, script def should know nothing about them: they may tweak some 3d-party tfm config

        paramsMap := map[string]any{}
        if jsonOrYamlBytesParams != nil {
                scriptParamsType, err := getFileType(scriptParamsUrl)
                if err != nil {
                        return nil, ScriptInitContentProblem, err
                }
                if err := JsonOrYamlUnmarshal(scriptParamsType, jsonOrYamlBytesParams, &paramsMap); err != nil {
                        return nil, ScriptInitContentProblem, fmt.Errorf("cannot unmarshal script params from [%s]: [%s]", scriptParamsUrl, err.Error())
                }
        }

        replacerStrings := make([]string, len(paramsMap)*2)
        i := 0
        for templateParam, templateParamVal := range paramsMap {
                switch typedParamVal := templateParamVal.(type) {
                case string:
                        // Revert \n unescaping in parameter values - we want to preserve "\n"
                        if strings.Contains(typedParamVal, "\n") {
                                typedParamVal = strings.ReplaceAll(typedParamVal, "\n", "\\n")
                        }
                        // Just replace {param_name|string} with value, pay no attention to double quotes
                        replacerStrings[i] = fmt.Sprintf("{%s|string}", templateParam)
                        replacerStrings[i+1] = typedParamVal
                case float64:
                        // Expect enclosed in double quotes
                        replacerStrings[i] = fmt.Sprintf(`"{%s|number}"`, templateParam)
                        if typedParamVal == float64(int64(typedParamVal)) {
                                // It's an int in JSON
                                replacerStrings[i+1] = fmt.Sprintf("%d", int64(typedParamVal))
                        } else {
                                // It's a float in JSON
                                replacerStrings[i+1] = fmt.Sprintf("%f", typedParamVal)
                        }
                case bool:
                        // Expect enclosed in double quotes
                        replacerStrings[i] = fmt.Sprintf(`"{%s|bool}"`, templateParam)
                        replacerStrings[i+1] = fmt.Sprintf("%t", typedParamVal)
                default:
                        arrayParamVal, ok := templateParamVal.([]any)
                        if !ok {
                                return nil, ScriptInitContentProblem, fmt.Errorf("unsupported parameter type %T from [%s]: %s", templateParamVal, scriptParamsUrl, templateParam)
                        }
                        switch arrayParamVal[0].(type) {
                        case string:
                                // It's a stringlist
                                replacerStrings[i] = fmt.Sprintf(`"{%s|stringlist}"`, templateParam)
                                strArray := make([]string, len(arrayParamVal))
                                for i, itemAny := range arrayParamVal {
                                        itemStr, ok := itemAny.(string)
                                        if !ok {
                                                return nil, ScriptInitContentProblem, fmt.Errorf("stringlist contains non-string value type %T from [%s]: %s", itemAny, scriptParamsUrl, templateParam)
                                        }
                                        strArray[i] = fmt.Sprintf(`"%s"`, itemStr)
                                }
                                replacerStrings[i+1] = fmt.Sprintf("[%s]", strings.Join(strArray, ","))
                        default:
                                return nil, ScriptInitContentProblem, fmt.Errorf("unsupported array parameter type %T from [%s]: %s", arrayParamVal, scriptParamsUrl, templateParam)
                        }
                }
                i += 2
        }
        jsonOrYamlScriptString = strings.NewReplacer(replacerStrings...).Replace(jsonOrYamlScriptString)

        // Verify all parameters were replaced
        re = regexp.MustCompile(`({[a-zA-Z0-9_]+\|(string|number|bool)})`)
        unresolvedParamRefs := re.FindAllString(jsonOrYamlScriptString, -1)
        unresolvedParamMap := map[string]struct{}{}
        reservedParamRefs := map[string]struct{}{ReservedParamBatchIdx: {}}
        for _, paramRef := range unresolvedParamRefs {
                if _, ok := reservedParamRefs[paramRef]; !ok {
                        unresolvedParamMap[paramRef] = struct{}{}
                }
        }
        if len(unresolvedParamMap) > 0 {
                return nil, ScriptInitContentProblem, fmt.Errorf("unresolved parameter references in [%s]: %v; make sure that type in the script matches the type of the parameter value in the script parameters file", scriptUrl, unresolvedParamMap)
        }

        scriptType, err := getFileType(scriptUrl)
        if err != nil {
                return nil, ScriptInitContentProblem, err
        }

        newScript := &ScriptDef{}
        if err := newScript.Deserialize([]byte(jsonOrYamlScriptString), scriptType, customProcessorDefFactoryInstance, customProcessorsSettings, caPath, privateKeys); err != nil {
                return nil, ScriptInitContentProblem, fmt.Errorf("cannot deserialize script %s(%s): %s", scriptUrl, scriptParamsUrl, err.Error())
        }

        return newScript, ScriptInitNoProblem, nil
}

1	package sc
2
3	import (
4	"encoding/json"
5	"fmt"
6	"regexp"
7	"strings"
8	)
9
10	type ScriptInitProblemType int
11
12	const ScriptInitNoProblem ScriptInitProblemType = 0
13	const ScriptInitUrlProblem ScriptInitProblemType = 1
14	const ScriptInitContentProblem ScriptInitProblemType = 2
15	const ScriptInitConnectivityProblem ScriptInitProblemType = 3
16
17	func getFileType(url string) (ScriptType, error) {	1✔
18	if strings.HasSuffix(url, ".json") {	2✔
19	return ScriptJson, nil	1✔
20	} else if strings.HasSuffix(url, ".yaml") {	1✔
21	return ScriptYaml, nil	×
22	}	×
23	return ScriptUnknown, fmt.Errorf("cannot detect json/yaml file type: %s", url)	×
24	}
25
26	func NewScriptFromFileBytes(
27	caPath string,
28	privateKeys map[string]string,
29	scriptUrl string,
30	jsonOrYamlBytesScript []byte,
31	scriptParamsUrl string,
32	jsonOrYamlBytesParams []byte,
33	customProcessorDefFactoryInstance CustomProcessorDefFactory,
34	customProcessorsSettings map[string]json.RawMessage) (*ScriptDef, ScriptInitProblemType, error) {	1✔
35	// Make sure parameters are in canonical format: {param_name\|param_type}	1✔
36	jsonOrYamlScriptString := string(jsonOrYamlBytesScript)	1✔
37		1✔
38	// Default param type is string: {param} -> {param\|string}	1✔
39	re := regexp.MustCompile("{[ ]([a-zA-Z0-9_]+)[ ]}")	1✔
40	jsonOrYamlScriptString = re.ReplaceAllString(jsonOrYamlScriptString, "{$1\|string}")	1✔
41		1✔
42	// Remove spaces: { param_name \| param_type } -> {param_name\|param_type}	1✔
43	re = regexp.MustCompile(`{[ ]([a-zA-Z0-9_]+)[ ]\\|[ ](string\|number\|bool\|stringlist)[ ]}`)	1✔
44	jsonOrYamlScriptString = re.ReplaceAllString(jsonOrYamlScriptString, "{$1\|$2}")	1✔
45		1✔
46	// Verify that number/bool must be like "{param_name\|number}", no extra characters between double quotes and curly braces	1✔
47	// This is a big limitation of the existing parameter implementation.	1✔
48	// Those double quotes must be there in order to keep JSON well-formed.	1✔
49	// Because of this, using number/bool parameters in strings is not allowed, use string paramater like "10" and "false" in those cases.	1✔
50	re = regexp.MustCompile(`([^"]{[a-zA-Z0-9_]+\\|(number\|bool)})\|({[a-zA-Z0-9_]+\\|(number\|bool)}[^"])`)	1✔
51	invalidParamRefs := re.FindAllString(jsonOrYamlScriptString, -1)	1✔
52	if len(invalidParamRefs) > 0 {	1✔
53	return nil, ScriptInitUrlProblem, fmt.Errorf("cannot parse number/bool script parameter references in [%s], the following parameter references should not have extra characters between curly braces and double quotes: [%s]", scriptUrl, strings.Join(invalidParamRefs, ","))	×
54	}	×
55
56	// Apply template params here, script def should know nothing about them: they may tweak some 3d-party tfm config
57
58	paramsMap := map[string]any{}	1✔
59	if jsonOrYamlBytesParams != nil {	2✔
60	scriptParamsType, err := getFileType(scriptParamsUrl)	1✔
61	if err != nil {	1✔
62	return nil, ScriptInitContentProblem, err	×
63	}	×
64	if err := JsonOrYamlUnmarshal(scriptParamsType, jsonOrYamlBytesParams, &paramsMap); err != nil {	1✔
65	return nil, ScriptInitContentProblem, fmt.Errorf("cannot unmarshal script params from [%s]: [%s]", scriptParamsUrl, err.Error())	×
66	}	×
67	}
68
69	replacerStrings := make([]string, len(paramsMap)*2)	1✔
70	i := 0	1✔
71	for templateParam, templateParamVal := range paramsMap {	2✔
72	switch typedParamVal := templateParamVal.(type) {	1✔
73	case string:	1✔
74	// Revert \n unescaping in parameter values - we want to preserve "\n"	1✔
75	if strings.Contains(typedParamVal, "\n") {	2✔
76	typedParamVal = strings.ReplaceAll(typedParamVal, "\n", "\\n")	1✔
77	}	1✔
78	// Just replace {param_name\|string} with value, pay no attention to double quotes
79	replacerStrings[i] = fmt.Sprintf("{%s\|string}", templateParam)	1✔
80	replacerStrings[i+1] = typedParamVal	1✔
81	case float64:	1✔
82	// Expect enclosed in double quotes	1✔
83	replacerStrings[i] = fmt.Sprintf(`"{%s\|number}"`, templateParam)	1✔
84	if typedParamVal == float64(int64(typedParamVal)) {	2✔
85	// It's an int in JSON	1✔
86	replacerStrings[i+1] = fmt.Sprintf("%d", int64(typedParamVal))	1✔
87	} else {	1✔
88	// It's a float in JSON	×
89	replacerStrings[i+1] = fmt.Sprintf("%f", typedParamVal)	×
90	}	×
91	case bool:	1✔
92	// Expect enclosed in double quotes	1✔
93	replacerStrings[i] = fmt.Sprintf(`"{%s\|bool}"`, templateParam)	1✔
94	replacerStrings[i+1] = fmt.Sprintf("%t", typedParamVal)	1✔
95	default:	1✔
96	arrayParamVal, ok := templateParamVal.([]any)	1✔
97	if !ok {	1✔
NEW 98	return nil, ScriptInitContentProblem, fmt.Errorf("unsupported parameter type %T from [%s]: %s", templateParamVal, scriptParamsUrl, templateParam)	×
NEW 99	}	×
100	switch arrayParamVal[0].(type) {	1✔
101	case string:	1✔
102	// It's a stringlist	1✔
103	replacerStrings[i] = fmt.Sprintf(`"{%s\|stringlist}"`, templateParam)	1✔
104	strArray := make([]string, len(arrayParamVal))	1✔
105	for i, itemAny := range arrayParamVal {	2✔
106	itemStr, ok := itemAny.(string)	1✔
107	if !ok {	1✔
NEW 108	return nil, ScriptInitContentProblem, fmt.Errorf("stringlist contains non-string value type %T from [%s]: %s", itemAny, scriptParamsUrl, templateParam)	×
UNCOV 109	}	×
110	strArray[i] = fmt.Sprintf(`"%s"`, itemStr)	1✔
111	}
112	replacerStrings[i+1] = fmt.Sprintf("[%s]", strings.Join(strArray, ","))	1✔
NEW 113	default:	×
NEW 114	return nil, ScriptInitContentProblem, fmt.Errorf("unsupported array parameter type %T from [%s]: %s", arrayParamVal, scriptParamsUrl, templateParam)	×
115	}
116	}
117	i += 2	1✔
118	}
119	jsonOrYamlScriptString = strings.NewReplacer(replacerStrings...).Replace(jsonOrYamlScriptString)	1✔
120		1✔
121	// Verify all parameters were replaced	1✔
122	re = regexp.MustCompile(`({[a-zA-Z0-9_]+\\|(string\|number\|bool)})`)	1✔
123	unresolvedParamRefs := re.FindAllString(jsonOrYamlScriptString, -1)	1✔
124	unresolvedParamMap := map[string]struct{}{}	1✔
125	reservedParamRefs := map[string]struct{}{ReservedParamBatchIdx: {}}	1✔
126	for _, paramRef := range unresolvedParamRefs {	2✔
127	if _, ok := reservedParamRefs[paramRef]; !ok {	2✔
128	unresolvedParamMap[paramRef] = struct{}{}	1✔
129	}	1✔
130	}
131	if len(unresolvedParamMap) > 0 {	2✔
132	return nil, ScriptInitContentProblem, fmt.Errorf("unresolved parameter references in [%s]: %v; make sure that type in the script matches the type of the parameter value in the script parameters file", scriptUrl, unresolvedParamMap)	1✔
133	}	1✔
134
135	scriptType, err := getFileType(scriptUrl)	1✔
136	if err != nil {	1✔
137	return nil, ScriptInitContentProblem, err	×
138	}	×
139
140	newScript := &ScriptDef{}	1✔
141	if err := newScript.Deserialize([]byte(jsonOrYamlScriptString), scriptType, customProcessorDefFactoryInstance, customProcessorsSettings, caPath, privateKeys); err != nil {	2✔
142	return nil, ScriptInitContentProblem, fmt.Errorf("cannot deserialize script %s(%s): %s", scriptUrl, scriptParamsUrl, err.Error())	1✔
143	}	1✔
144
145	return newScript, ScriptInitNoProblem, nil	1✔
146	}

capillariesio / capillaries / 14208558196

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous