15982590714

Committed 30 Jun 2025 08:07PM UTC coverage: 57.402% (+0.01%) from 57.392%

Build # 15982590714

Build Type

Pull #5702

github

Committed by

web-flow

Commit Message

Merge e8afcef43 into 9b4f171a0

Pull Request Pull Request #5702: Add templates for REST ingest and remediate, and YQ remediation

Run Details

63 of 78 new or added lines in 7 files covered. (80.77%)

3 existing lines in 2 files now uncovered.

18596 of 32396 relevant lines covered (57.4%)

37.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.09

/internal/engine/ingester/rest/rest.go

// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
// SPDX-License-Identifier: Apache-2.0

// Package rest provides the REST rule data ingest engine
package rest

import (
        "bytes"
        "cmp"
        "context"
        "encoding/json"
        "errors"
        "fmt"
        "io"
        "net/http"
        "strings"

        "github.com/google/go-github/v63/github"
        "github.com/rs/zerolog"
        "github.com/rs/zerolog/log"
        "google.golang.org/protobuf/reflect/protoreflect"

        "github.com/mindersec/minder/internal/util"
        pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
        "github.com/mindersec/minder/pkg/engine/v1/interfaces"
        "github.com/mindersec/minder/pkg/entities/v1/checkpoints"
)

const (
        // RestRuleDataIngestType is the type of the REST rule data ingest engine
        RestRuleDataIngestType = "rest"

        // MaxBytesLimit is the maximum number of bytes to read from the response body
        // We limit to 1MB to prevent abuse
        MaxBytesLimit int64 = 1 << 20
        // endpointBytesLimit is the maximum number of bytes for the endpoint
        endpointBytesLimit = 1024
        // bodyBytesLimit is the maximum number of bytes for the body
        bodyBytesLimit = 1024
        // methodBytesLimit is the maximum number of bytes for the method
        methodBytesLimit = 10
)

type ingestorFallback struct {
        // httpCode is the HTTP status code to return
        httpCode int
        // Body is the body to return
        body string
}

// Ingestor is the engine for a rule type that uses REST data ingest
type Ingestor struct {
        restCfg          *pb.RestType
        cli              interfaces.RESTProvider
        endpointTemplate *util.SafeTemplate
        bodyTemplate     *util.SafeTemplate
        methodTemplate   *util.SafeTemplate
        fallback         []ingestorFallback
}

// NewRestRuleDataIngest creates a new REST rule data ingest engine
func NewRestRuleDataIngest(
        restCfg *pb.RestType,
        cli interfaces.RESTProvider,
) (*Ingestor, error) {
        if len(restCfg.Endpoint) == 0 {
                return nil, fmt.Errorf("missing endpoint")
        }

        tmpl, err := util.NewSafeTextTemplate(&restCfg.Endpoint, "endpoint")
        if err != nil {
                return nil, fmt.Errorf("cannot parse endpoint template: %w", err)
        }

        var bodyTmpl *util.SafeTemplate
        if restCfg.GetBody() != "" {
                bodyTmpl, err = util.NewSafeHTMLTemplate(restCfg.Body, "body")
                if err != nil {
                        return nil, fmt.Errorf("cannot parse body template: %w", err)
                }
        }

        method := cmp.Or(restCfg.Method, http.MethodGet)
        methodTmpl, err := util.NewSafeTextTemplate(&method, "method")
        if err != nil {
                return nil, fmt.Errorf("cannot parse method template: %w", err)
        }

        fallback := make([]ingestorFallback, len(restCfg.Fallback))
        for _, fb := range restCfg.Fallback {
                fb := fb
                fallback = append(fallback, ingestorFallback{
                        httpCode: int(fb.HttpCode),
                        body:     fb.Body,
                })
        }

        return &Ingestor{
                restCfg:          restCfg,
                cli:              cli,
                endpointTemplate: tmpl,
                bodyTemplate:     bodyTmpl,
                methodTemplate:   methodTmpl,
                fallback:         fallback,
        }, nil
}

// EndpointTemplateParams is the parameters for the REST endpoint template
type EndpointTemplateParams struct {
        // Entity is the entity to be evaluated
        Entity any
        // Params are the parameters to be used in the template
        Params map[string]any
}

// GetType returns the type of the REST rule data ingest engine
func (*Ingestor) GetType() string {
        return RestRuleDataIngestType
}

// GetConfig returns the config for the REST rule data ingest engine
func (rdi *Ingestor) GetConfig() protoreflect.ProtoMessage {
        return rdi.restCfg
}

// Ingest calls the REST endpoint and returns the data
func (rdi *Ingestor) Ingest(
        ctx context.Context, ent protoreflect.ProtoMessage, params map[string]any,
) (*interfaces.Ingested, error) {
        retp := &EndpointTemplateParams{
                Entity: ent,
                Params: params,
        }

        endpoint, err := rdi.endpointTemplate.Render(ctx, retp, endpointBytesLimit)
        if err != nil {
                return nil, fmt.Errorf("cannot execute endpoint template: %w", err)
        }

        var bodyOut any
        if rdi.bodyTemplate != nil {
                var body bytes.Buffer
                if err := rdi.bodyTemplate.Execute(ctx, &body, retp, bodyBytesLimit); err != nil {
                        return nil, fmt.Errorf("cannot execute body template: %w", err)
                }
                // Newlines are not valid in JSON, but are handy when writing e.g. graphql queries.
                data := bytes.ReplaceAll(body.Bytes(), []byte("\n"), []byte(" "))
                if err := json.Unmarshal(data, &bodyOut); err != nil {
                        return nil, fmt.Errorf("cannot parse request body as JSON: %w", err)
                }
        }

        method, err := rdi.methodTemplate.Render(ctx, retp, methodBytesLimit)
        if err != nil {
                return nil, fmt.Errorf("cannot execute method template: %w", err)
        }
        method = strings.ToUpper(method)

        req, err := rdi.cli.NewRequest(method, endpoint, bodyOut)
        if err != nil {
                return nil, fmt.Errorf("cannot create request: %w", err)
        }

        respRdr, err := rdi.doRequest(ctx, req)
        if err != nil {
                return nil, fmt.Errorf("cannot do request: %w", err)
        }

        defer func() {
                if err := respRdr.Close(); err != nil {
                        log.Printf("cannot close response body: %v", err)
                }
        }()

        data, err := rdi.parseBody(respRdr)
        if err != nil {
                return nil, fmt.Errorf("cannot parse body: %w", err)
        }

        return &interfaces.Ingested{
                Object:     data,
                Checkpoint: checkpoints.NewCheckpointV1Now().WithHTTP(endpoint, method),
        }, nil
}

func (rdi *Ingestor) doRequest(ctx context.Context, req *http.Request) (io.ReadCloser, error) {
        resp, err := rdi.cli.Do(ctx, req)
        if err == nil {
                // Early-exit on success
                return resp.Body, nil
        }

        if fallbackBody := errorToFallback(err, rdi.fallback); fallbackBody != nil {
                // the go-github REST API has a funny way of returning HTTP status codes,
                // on a non-200 status it will return a github.ErrorResponse
                // whereas the standard library will return nil error and the HTTP status code in the response
                return fallbackBody, nil
        }

        return nil, fmt.Errorf("cannot make request: %w", err)
}

func errorToFallback(err error, fallback []ingestorFallback) io.ReadCloser {
        var respErr *github.ErrorResponse
        if errors.As(err, &respErr) {
                if respErr.Response != nil {
                        return httpStatusToFallback(respErr.Response.StatusCode, fallback)
                }
        }

        return nil
}

func httpStatusToFallback(httpStatus int, fallback []ingestorFallback) io.ReadCloser {
        for _, fb := range fallback {
                if fb.httpCode == httpStatus {
                        zerolog.Ctx(context.Background()).Debug().Msgf("falling back to body [%s]", fb.body)
                        return io.NopCloser(strings.NewReader(fb.body))
                }
        }

        return nil
}

func (rdi *Ingestor) parseBody(body io.Reader) (any, error) {
        var data any
        var err error

        if body == nil {
                return nil, nil
        }

        lr := io.LimitReader(body, MaxBytesLimit)

        if rdi.restCfg.Parse == "json" {
                var jsonData any
                dec := json.NewDecoder(lr)
                if err := dec.Decode(&jsonData); err != nil {
                        return nil, fmt.Errorf("cannot decode json: %w", err)
                }

                data = jsonData
        } else {
                data, err = io.ReadAll(lr)
                if err != nil {
                        return nil, fmt.Errorf("cannot read response body: %w", err)
                }
        }

        return data, nil
}

1	// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
2	// SPDX-License-Identifier: Apache-2.0
3
4	// Package rest provides the REST rule data ingest engine
5	package rest
6
7	import (
8	"bytes"
9	"cmp"
10	"context"
11	"encoding/json"
12	"errors"
13	"fmt"
14	"io"
15	"net/http"
16	"strings"
17
18	"github.com/google/go-github/v63/github"
19	"github.com/rs/zerolog"
20	"github.com/rs/zerolog/log"
21	"google.golang.org/protobuf/reflect/protoreflect"
22
23	"github.com/mindersec/minder/internal/util"
24	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
25	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
26	"github.com/mindersec/minder/pkg/entities/v1/checkpoints"
27	)
28
29	const (
30	// RestRuleDataIngestType is the type of the REST rule data ingest engine
31	RestRuleDataIngestType = "rest"
32
33	// MaxBytesLimit is the maximum number of bytes to read from the response body
34	// We limit to 1MB to prevent abuse
35	MaxBytesLimit int64 = 1 << 20
36	// endpointBytesLimit is the maximum number of bytes for the endpoint
37	endpointBytesLimit = 1024
38	// bodyBytesLimit is the maximum number of bytes for the body
39	bodyBytesLimit = 1024
40	// methodBytesLimit is the maximum number of bytes for the method
41	methodBytesLimit = 10
42	)
43
44	type ingestorFallback struct {
45	// httpCode is the HTTP status code to return
46	httpCode int
47	// Body is the body to return
48	body string
49	}
50
51	// Ingestor is the engine for a rule type that uses REST data ingest
52	type Ingestor struct {
53	restCfg *pb.RestType
54	cli interfaces.RESTProvider
55	endpointTemplate *util.SafeTemplate
56	bodyTemplate *util.SafeTemplate
57	methodTemplate *util.SafeTemplate
58	fallback []ingestorFallback
59	}
60
61	// NewRestRuleDataIngest creates a new REST rule data ingest engine
62	func NewRestRuleDataIngest(
63	restCfg *pb.RestType,
64	cli interfaces.RESTProvider,
65	) (*Ingestor, error) {	8✔
66	if len(restCfg.Endpoint) == 0 {	9✔
67	return nil, fmt.Errorf("missing endpoint")	1✔
68	}	1✔
69
70	tmpl, err := util.NewSafeTextTemplate(&restCfg.Endpoint, "endpoint")	7✔
71	if err != nil {	8✔
72	return nil, fmt.Errorf("cannot parse endpoint template: %w", err)	1✔
73	}	1✔
74
75	var bodyTmpl *util.SafeTemplate	6✔
76	if restCfg.GetBody() != "" {	7✔
77	bodyTmpl, err = util.NewSafeHTMLTemplate(restCfg.Body, "body")	1✔
78	if err != nil {	1✔
NEW 79	return nil, fmt.Errorf("cannot parse body template: %w", err)	×
NEW 80	}	×
81	}
82
83	method := cmp.Or(restCfg.Method, http.MethodGet)	6✔
84	methodTmpl, err := util.NewSafeTextTemplate(&method, "method")	6✔
85	if err != nil {	6✔
NEW 86	return nil, fmt.Errorf("cannot parse method template: %w", err)	×
NEW 87	}	×
88
89	fallback := make([]ingestorFallback, len(restCfg.Fallback))	6✔
90	for _, fb := range restCfg.Fallback {	7✔
91	fb := fb	1✔
92	fallback = append(fallback, ingestorFallback{	1✔
93	httpCode: int(fb.HttpCode),	1✔
94	body: fb.Body,	1✔
95	})	1✔
96	}	1✔
97
98	return &Ingestor{	6✔
99	restCfg: restCfg,	6✔
100	cli: cli,	6✔
101	endpointTemplate: tmpl,	6✔
102	bodyTemplate: bodyTmpl,	6✔
103	methodTemplate: methodTmpl,	6✔
104	fallback: fallback,	6✔
105	}, nil	6✔
106	}
107
108	// EndpointTemplateParams is the parameters for the REST endpoint template
109	type EndpointTemplateParams struct {
110	// Entity is the entity to be evaluated
111	Entity any
112	// Params are the parameters to be used in the template
113	Params map[string]any
114	}
115
116	// GetType returns the type of the REST rule data ingest engine
117	func (*Ingestor) GetType() string {	6✔
118	return RestRuleDataIngestType	6✔
119	}	6✔
120
121	// GetConfig returns the config for the REST rule data ingest engine
122	func (rdi *Ingestor) GetConfig() protoreflect.ProtoMessage {	12✔
123	return rdi.restCfg	12✔
124	}	12✔
125
126	// Ingest calls the REST endpoint and returns the data
127	func (rdi *Ingestor) Ingest(
128	ctx context.Context, ent protoreflect.ProtoMessage, params map[string]any,
129	) (*interfaces.Ingested, error) {	4✔
130	retp := &EndpointTemplateParams{	4✔
131	Entity: ent,	4✔
132	Params: params,	4✔
133	}	4✔
134		4✔
135	endpoint, err := rdi.endpointTemplate.Render(ctx, retp, endpointBytesLimit)	4✔
136	if err != nil {	4✔
137	return nil, fmt.Errorf("cannot execute endpoint template: %w", err)	×
138	}	×
139
140	var bodyOut any	4✔
141	if rdi.bodyTemplate != nil {	5✔
142	var body bytes.Buffer	1✔
143	if err := rdi.bodyTemplate.Execute(ctx, &body, retp, bodyBytesLimit); err != nil {	1✔
NEW 144	return nil, fmt.Errorf("cannot execute body template: %w", err)	×
NEW 145	}	×
146	// Newlines are not valid in JSON, but are handy when writing e.g. graphql queries.
147	data := bytes.ReplaceAll(body.Bytes(), []byte("\n"), []byte(" "))	1✔
148	if err := json.Unmarshal(data, &bodyOut); err != nil {	1✔
NEW 149	return nil, fmt.Errorf("cannot parse request body as JSON: %w", err)	×
NEW 150	}	×
151	}
152
153	method, err := rdi.methodTemplate.Render(ctx, retp, methodBytesLimit)	4✔
154	if err != nil {	4✔
NEW 155	return nil, fmt.Errorf("cannot execute method template: %w", err)	×
UNCOV 156	}	×
157	method = strings.ToUpper(method)	4✔
158		4✔
159	req, err := rdi.cli.NewRequest(method, endpoint, bodyOut)	4✔
160	if err != nil {	4✔
161	return nil, fmt.Errorf("cannot create request: %w", err)	×
162	}	×
163
164	respRdr, err := rdi.doRequest(ctx, req)	4✔
165	if err != nil {	5✔
166	return nil, fmt.Errorf("cannot do request: %w", err)	1✔
167	}	1✔
168
169	defer func() {	6✔
170	if err := respRdr.Close(); err != nil {	3✔
171	log.Printf("cannot close response body: %v", err)	×
172	}	×
173	}()
174
175	data, err := rdi.parseBody(respRdr)	3✔
176	if err != nil {	3✔
177	return nil, fmt.Errorf("cannot parse body: %w", err)	×
178	}	×
179
180	return &interfaces.Ingested{	3✔
181	Object: data,	3✔
182	Checkpoint: checkpoints.NewCheckpointV1Now().WithHTTP(endpoint, method),	3✔
183	}, nil	3✔
184	}
185
186	func (rdi Ingestor) doRequest(ctx context.Context, req http.Request) (io.ReadCloser, error) {	4✔
187	resp, err := rdi.cli.Do(ctx, req)	4✔
188	if err == nil {	6✔
189	// Early-exit on success	2✔
190	return resp.Body, nil	2✔
191	}	2✔
192
193	if fallbackBody := errorToFallback(err, rdi.fallback); fallbackBody != nil {	3✔
194	// the go-github REST API has a funny way of returning HTTP status codes,	1✔
195	// on a non-200 status it will return a github.ErrorResponse	1✔
196	// whereas the standard library will return nil error and the HTTP status code in the response	1✔
197	return fallbackBody, nil	1✔
198	}	1✔
199
200	return nil, fmt.Errorf("cannot make request: %w", err)	1✔
201	}
202
203	func errorToFallback(err error, fallback []ingestorFallback) io.ReadCloser {	2✔
204	var respErr *github.ErrorResponse	2✔
205	if errors.As(err, &respErr) {	4✔
206	if respErr.Response != nil {	4✔
207	return httpStatusToFallback(respErr.Response.StatusCode, fallback)	2✔
208	}	2✔
209	}
210
211	return nil	×
212	}
213
214	func httpStatusToFallback(httpStatus int, fallback []ingestorFallback) io.ReadCloser {	2✔
215	for _, fb := range fallback {	4✔
216	if fb.httpCode == httpStatus {	3✔
217	zerolog.Ctx(context.Background()).Debug().Msgf("falling back to body [%s]", fb.body)	1✔
218	return io.NopCloser(strings.NewReader(fb.body))	1✔
219	}	1✔
220	}
221
222	return nil	1✔
223	}
224
225	func (rdi *Ingestor) parseBody(body io.Reader) (any, error) {	5✔
226	var data any	5✔
227	var err error	5✔
228		5✔
229	if body == nil {	5✔
230	return nil, nil	×
231	}	×
232
233	lr := io.LimitReader(body, MaxBytesLimit)	5✔
234		5✔
235	if rdi.restCfg.Parse == "json" {	9✔
236	var jsonData any	4✔
237	dec := json.NewDecoder(lr)	4✔
238	if err := dec.Decode(&jsonData); err != nil {	5✔
239	return nil, fmt.Errorf("cannot decode json: %w", err)	1✔
240	}	1✔
241
242	data = jsonData	3✔
243	} else {	1✔
244	data, err = io.ReadAll(lr)	1✔
245	if err != nil {	1✔
246	return nil, fmt.Errorf("cannot read response body: %w", err)	×
247	}	×
248	}
249
250	return data, nil	4✔
251	}

mindersec / minder / 15982590714

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous