15982590714

Committed 30 Jun 2025 08:07PM UTC coverage: 57.402% (+0.01%) from 57.392%

Build # 15982590714

Build Type

Pull #5702

github

Committed by

web-flow

Commit Message

Merge e8afcef43 into 9b4f171a0

Pull Request Pull Request #5702: Add templates for REST ingest and remediate, and YQ remediation

Run Details

63 of 78 new or added lines in 7 files covered. (80.77%)

3 existing lines in 2 files now uncovered.

18596 of 32396 relevant lines covered (57.4%)

37.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.57

/internal/engine/actions/remediate/pull_request/types_yq.go

// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
// SPDX-License-Identifier: Apache-2.0

package pull_request

import (
        "bytes"
        "context"
        "encoding/json"
        "fmt"

        "github.com/go-git/go-billy/v5/util"
        "github.com/go-git/go-git/v5/plumbing/filemode"
        "github.com/mikefarah/yq/v4/pkg/yqlib"
        "github.com/rs/zerolog"
        "google.golang.org/protobuf/proto"
        gologging "gopkg.in/op/go-logging.v1"

        "github.com/mindersec/minder/internal/engine/interfaces"
        textutil "github.com/mindersec/minder/internal/util"
)

// TODO: document the YQ remediation model.

// unfortunately yqlib does seem to be using global variables...
func init() {
        // setting the log level to critical pretty much silences the logging
        gologging.SetLevel(gologging.CRITICAL, yqLibModule)
        yqlib.InitExpressionParser()
}

type patternType string

const (
        patternTypeGlob patternType = "glob"

        maxExpressionSize int = 3000
)

const (
        yqLibModule = "yq-lib"
)

var _ fsModifier = (*yqExecute)(nil)

type yqExecuteConfig struct {
        Expression string `json:"expression"`
        Patterns   []struct {
                Pattern string `json:"pattern"`
                Type    string `json:"type"`
        }
}

type yqExecute struct {
        fsChangeSet

        config yqExecuteConfig
}

var _ modificationConstructor = newYqExecute

func newYqExecute(
        params *modificationConstructorParams,
) (fsModifier, error) {

        confMap := make(map[string]any)
        if params.prCfg.GetParams() != nil {
                confMap = params.prCfg.Params.AsMap()
        }

        rawConfig, err := json.Marshal(confMap)
        if err != nil {
                return nil, fmt.Errorf("cannot marshal config")
        }

        var conf yqExecuteConfig
        err = json.Unmarshal(rawConfig, &conf)
        if err != nil {
                return nil, fmt.Errorf("cannot marshal config")
        }

        return &yqExecute{
                fsChangeSet: fsChangeSet{
                        fs: params.bfs,
                },

                config: conf,
        }, nil
}

// templateParams are the parameters for template expansion for the expression template
type templateParams struct {
        // Entity is the metadata for the entity which was evaluated
        Entity any
        // Profile is the parameters to be used in the template
        Profile map[string]any
        // Params are the rule instance parameters
        Params map[string]any
        // EvalResultOutput is the output from the rule evaluation
        EvalResultOutput any
}

func (yq *yqExecute) createFsModEntries(ctx context.Context, ent proto.Message, params interfaces.ActionsParams) error {
        matchingFiles := make([]string, 0)
        for _, pattern := range yq.config.Patterns {
                if pattern.Type != string(patternTypeGlob) {
                        zerolog.Ctx(ctx).
                                Warn().
                                Str("pattern.Type", pattern.Type).
                                Msg("unsupported pattern type")
                        continue
                }

                patternMatches, err := util.Glob(yq.fs, pattern.Pattern)
                if err != nil {
                        return fmt.Errorf("cannot get matching files: %w", err)
                }
                matchingFiles = append(matchingFiles, patternMatches...)
        }

        templateData := templateParams{
                Entity:  ent,
                Profile: params.GetRule().Def,
                Params:  params.GetRule().Params,
        }
        if params.GetEvalResult() != nil {
                templateData.EvalResultOutput = params.GetEvalResult().Output
        }

        expression := ""
        if yq.config.Expression != "" {
                expressionBytes := new(bytes.Buffer)
                expressionTmpl, err := textutil.NewSafeTextTemplate(&yq.config.Expression, "expression")
                if err != nil {
                        return fmt.Errorf("unable to parse templates in expression: %w", err)
                }
                if err := expressionTmpl.Execute(ctx, expressionBytes, templateData, maxExpressionSize); err != nil {
                        return fmt.Errorf("unable to expand templates in expression: %w", err)
                }
                expression = expressionBytes.String()
        }

        for _, file := range matchingFiles {
                newContent, err := yq.executeYq(file, expression)
                if err != nil {
                        return fmt.Errorf("cannot execute yq: %w", err)
                }
                yq.entries = append(yq.entries, &fsEntry{
                        Path:    file,
                        Content: newContent,
                        Mode:    filemode.Regular.String(),
                })
        }

        return nil
}

func (yq *yqExecute) modifyFs() ([]*fsEntry, error) {
        err := yq.writeEntries()
        if err != nil {
                return nil, fmt.Errorf("cannot write entries: %w", err)
        }
        return yq.entries, nil
}

func (yq *yqExecute) executeYq(filename, expression string) (string, error) {
        file, err := yq.fs.Open(filename)
        if err != nil {
                return "", fmt.Errorf("cannot read file: %w", err)
        }

        out := new(bytes.Buffer)
        encoder := yqlib.NewYamlEncoder(yqlib.NewDefaultYamlPreferences())
        printer := yqlib.NewPrinter(encoder, yqlib.NewSinglePrinterWriter(out))

        expParser := yqlib.ExpressionParser
        expressionNode, err := expParser.ParseExpression(expression)
        if err != nil {
                return "", fmt.Errorf("cannot parse expression: %w", err)
        }

        decoder := yqlib.NewYamlDecoder(yqlib.NewDefaultYamlPreferences())
        parser := yqlib.NewStreamEvaluator()
        _, err = parser.Evaluate(filename, file, expressionNode, printer, decoder)
        if err != nil {
                return "", fmt.Errorf("cannot evaluate expression: %w", err)
        }

        return out.String(), nil
}

1	// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
2	// SPDX-License-Identifier: Apache-2.0
3
4	package pull_request
5
6	import (
7	"bytes"
8	"context"
9	"encoding/json"
10	"fmt"
11
12	"github.com/go-git/go-billy/v5/util"
13	"github.com/go-git/go-git/v5/plumbing/filemode"
14	"github.com/mikefarah/yq/v4/pkg/yqlib"
15	"github.com/rs/zerolog"
16	"google.golang.org/protobuf/proto"
17	gologging "gopkg.in/op/go-logging.v1"
18
19	"github.com/mindersec/minder/internal/engine/interfaces"
20	textutil "github.com/mindersec/minder/internal/util"
21	)
22
23	// TODO: document the YQ remediation model.
24
25	// unfortunately yqlib does seem to be using global variables...
26	func init() {	20✔
27	// setting the log level to critical pretty much silences the logging	20✔
28	gologging.SetLevel(gologging.CRITICAL, yqLibModule)	20✔
29	yqlib.InitExpressionParser()	20✔
30	}	20✔
31
32	type patternType string
33
34	const (
35	patternTypeGlob patternType = "glob"
36
37	maxExpressionSize int = 3000
38	)
39
40	const (
41	yqLibModule = "yq-lib"
42	)
43
44	var _ fsModifier = (*yqExecute)(nil)
45
46	type yqExecuteConfig struct {
47	Expression string `json:"expression"`
48	Patterns []struct {
49	Pattern string `json:"pattern"`
50	Type string `json:"type"`
51	}
52	}
53
54	type yqExecute struct {
55	fsChangeSet
56
57	config yqExecuteConfig
58	}
59
60	var _ modificationConstructor = newYqExecute
61
62	func newYqExecute(
63	params *modificationConstructorParams,
64	) (fsModifier, error) {	6✔
65		6✔
66	confMap := make(map[string]any)	6✔
67	if params.prCfg.GetParams() != nil {	11✔
68	confMap = params.prCfg.Params.AsMap()	5✔
69	}	5✔
70
71	rawConfig, err := json.Marshal(confMap)	6✔
72	if err != nil {	6✔
73	return nil, fmt.Errorf("cannot marshal config")	×
74	}	×
75
76	var conf yqExecuteConfig	6✔
77	err = json.Unmarshal(rawConfig, &conf)	6✔
78	if err != nil {	6✔
79	return nil, fmt.Errorf("cannot marshal config")	×
80	}	×
81
82	return &yqExecute{	6✔
83	fsChangeSet: fsChangeSet{	6✔
84	fs: params.bfs,	6✔
85	},	6✔
86		6✔
87	config: conf,	6✔
88	}, nil	6✔
89	}
90
91	// templateParams are the parameters for template expansion for the expression template
92	type templateParams struct {
93	// Entity is the metadata for the entity which was evaluated
94	Entity any
95	// Profile is the parameters to be used in the template
96	Profile map[string]any
97	// Params are the rule instance parameters
98	Params map[string]any
99	// EvalResultOutput is the output from the rule evaluation
100	EvalResultOutput any
101	}
102
103	func (yq *yqExecute) createFsModEntries(ctx context.Context, ent proto.Message, params interfaces.ActionsParams) error {	6✔
104	matchingFiles := make([]string, 0)	6✔
105	for _, pattern := range yq.config.Patterns {	16✔
106	if pattern.Type != string(patternTypeGlob) {	10✔
107	zerolog.Ctx(ctx).	×
108	Warn().	×
109	Str("pattern.Type", pattern.Type).	×
110	Msg("unsupported pattern type")	×
111	continue	×
112	}
113
114	patternMatches, err := util.Glob(yq.fs, pattern.Pattern)	10✔
115	if err != nil {	10✔
116	return fmt.Errorf("cannot get matching files: %w", err)	×
117	}	×
118	matchingFiles = append(matchingFiles, patternMatches...)	10✔
119	}
120
121	templateData := templateParams{	6✔
122	Entity: ent,	6✔
123	Profile: params.GetRule().Def,	6✔
124	Params: params.GetRule().Params,	6✔
125	}	6✔
126	if params.GetEvalResult() != nil {	8✔
127	templateData.EvalResultOutput = params.GetEvalResult().Output	2✔
128	}	2✔
129
130	expression := ""	6✔
131	if yq.config.Expression != "" {	11✔
132	expressionBytes := new(bytes.Buffer)	5✔
133	expressionTmpl, err := textutil.NewSafeTextTemplate(&yq.config.Expression, "expression")	5✔
134	if err != nil {	5✔
NEW 135	return fmt.Errorf("unable to parse templates in expression: %w", err)	×
NEW 136	}	×
137	if err := expressionTmpl.Execute(ctx, expressionBytes, templateData, maxExpressionSize); err != nil {	5✔
NEW 138	return fmt.Errorf("unable to expand templates in expression: %w", err)	×
NEW 139	}	×
140	expression = expressionBytes.String()	5✔
141	}
142
143	for _, file := range matchingFiles {	15✔
144	newContent, err := yq.executeYq(file, expression)	9✔
145	if err != nil {	10✔
146	return fmt.Errorf("cannot execute yq: %w", err)	1✔
147	}	1✔
148	yq.entries = append(yq.entries, &fsEntry{	8✔
149	Path: file,	8✔
150	Content: newContent,	8✔
151	Mode: filemode.Regular.String(),	8✔
152	})	8✔
153	}
154
155	return nil	5✔
156	}
157
158	func (yq yqExecute) modifyFs() ([]fsEntry, error) {	6✔
159	err := yq.writeEntries()	6✔
160	if err != nil {	6✔
161	return nil, fmt.Errorf("cannot write entries: %w", err)	×
162	}	×
163	return yq.entries, nil	6✔
164	}
165
166	func (yq *yqExecute) executeYq(filename, expression string) (string, error) {	9✔
167	file, err := yq.fs.Open(filename)	9✔
168	if err != nil {	9✔
169	return "", fmt.Errorf("cannot read file: %w", err)	×
170	}	×
171
172	out := new(bytes.Buffer)	9✔
173	encoder := yqlib.NewYamlEncoder(yqlib.NewDefaultYamlPreferences())	9✔
174	printer := yqlib.NewPrinter(encoder, yqlib.NewSinglePrinterWriter(out))	9✔
175		9✔
176	expParser := yqlib.ExpressionParser	9✔
177	expressionNode, err := expParser.ParseExpression(expression)	9✔
178	if err != nil {	10✔
179	return "", fmt.Errorf("cannot parse expression: %w", err)	1✔
180	}	1✔
181
182	decoder := yqlib.NewYamlDecoder(yqlib.NewDefaultYamlPreferences())	8✔
183	parser := yqlib.NewStreamEvaluator()	8✔
184	_, err = parser.Evaluate(filename, file, expressionNode, printer, decoder)	8✔
185	if err != nil {	8✔
186	return "", fmt.Errorf("cannot evaluate expression: %w", err)	×
187	}	×
188
189	return out.String(), nil	8✔
190	}

mindersec / minder / 15982590714

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous