16654340601

Committed 31 Jul 2025 04:16PM UTC coverage: 57.482% (-0.009%) from 57.491%

Build # 16654340601

Build Type

push

github

Committed by

web-flow

Commit Message

Move a number of engine interfaces to pkg/ (#5788)

Run Details

8 of 15 new or added lines in 6 files covered. (53.33%)

5 existing lines in 2 files now uncovered.

18622 of 32396 relevant lines covered (57.48%)

37.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

56.31

/internal/engine/ingester/git/git.go

// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
// SPDX-License-Identifier: Apache-2.0

// Package git provides the git rule data ingest engine
package git

import (
        "cmp"
        "context"
        "errors"
        "fmt"

        "github.com/go-git/go-billy/v5"
        "github.com/go-git/go-git/v5/plumbing"
        "github.com/go-git/go-git/v5/storage"
        "github.com/go-viper/mapstructure/v2"
        "google.golang.org/protobuf/reflect/protoreflect"

        pbinternal "github.com/mindersec/minder/internal/proto"
        pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
        "github.com/mindersec/minder/pkg/engine/v1/interfaces"
        "github.com/mindersec/minder/pkg/entities/v1/checkpoints"
        provifv1 "github.com/mindersec/minder/pkg/providers/v1"
)

const (
        // GitRuleDataIngestType is the type of the git rule data ingest engine
        GitRuleDataIngestType = "git"
        defaultBranch         = "main"
)

// Git is the engine for a rule type that uses git data ingest
type Git struct {
        cfg     *pb.GitType
        gitprov interfaces.GitProvider
}

// NewGitIngester creates a new git rule data ingest engine
func NewGitIngester(cfg *pb.GitType, gitprov interfaces.GitProvider) (*Git, error) {
        if gitprov == nil {
                return nil, fmt.Errorf("provider is nil")
        }

        if cfg == nil {
                cfg = &pb.GitType{}
        }

        return &Git{
                cfg:     cfg,
                gitprov: gitprov,
        }, nil
}

// GetType returns the type of the git rule data ingest engine
func (*Git) GetType() string {
        return GitRuleDataIngestType
}

// GetConfig returns the config for the git rule data ingest engine
func (gi *Git) GetConfig() protoreflect.ProtoMessage {
        return gi.cfg
}

// Ingest does the actual data ingestion for a rule type by cloning a git repo
func (gi *Git) Ingest(ctx context.Context, ent protoreflect.ProtoMessage, params map[string]any) (*interfaces.Ingested, error) {
        switch entity := ent.(type) {
        case *pb.Repository:
                return gi.ingestRepository(ctx, entity, params)
        case *pbinternal.PullRequest:
                return gi.ingestPullRequest(ctx, entity, params)
        default:
                return nil, fmt.Errorf("git is only supported for repositories and pull requests")
        }
}

func (gi *Git) ingestRepository(ctx context.Context, repo *pb.Repository, params map[string]any) (*interfaces.Ingested, error) {
        userCfg := &IngesterConfig{}
        if err := mapstructure.Decode(params, userCfg); err != nil {
                return nil, fmt.Errorf("failed to read git ingester configuration from params: %w", err)
        }

        url := cmp.Or(userCfg.CloneURL, repo.GetCloneUrl())
        if url == "" {
                return nil, fmt.Errorf("could not get clone url")
        }

        branch := cmp.Or(userCfg.Branch, gi.cfg.Branch, repo.GetDefaultBranch(), defaultBranch)
        fs, storer, head, err := gi.fetchClone(ctx, url, branch)
        if err != nil {
                return nil, fmt.Errorf("failed to clone %s from %s: %w", branch, url, err)
        }

        hsh := head.Hash()

        chkpoint := checkpoints.NewCheckpointV1Now().
                WithBranch(branch).
                WithCommitHash(hsh.String())

        return &interfaces.Ingested{
                Object:     nil,
                Fs:         fs,
                Storer:     storer,
                Checkpoint: chkpoint,
        }, nil
}

func (gi *Git) ingestPullRequest(
        ctx context.Context, ent *pbinternal.PullRequest, params map[string]any) (*interfaces.Ingested, error) {
        // TODO: we don't actually have any configuration here.  Do we need to read the configuration?
        userCfg := &IngesterConfig{}
        if err := mapstructure.Decode(params, userCfg); err != nil {
                return nil, fmt.Errorf("failed to read git ingester configuration from params: %w", err)
        }

        if ent.GetBaseCloneUrl() == "" || ent.GetBaseRef() == "" {
                return nil, fmt.Errorf("could not get PR base branch %q from %q", ent.GetBaseRef(), ent.GetBaseCloneUrl())
        }
        if ent.GetTargetCloneUrl() == "" || ent.GetTargetRef() == "" {
                return nil, fmt.Errorf("could not get PR target branch %q from %q", ent.GetTargetRef(), ent.GetTargetCloneUrl())
        }

        baseFs, _, _, err := gi.fetchClone(ctx, ent.GetBaseCloneUrl(), ent.GetBaseRef())
        if err != nil {
                return nil, fmt.Errorf("failed to clone base branch %s from %s: %w", ent.GetBaseRef(), ent.GetBaseCloneUrl(), err)
        }
        targetFs, storer, head, err := gi.fetchClone(ctx, ent.GetTargetCloneUrl(), ent.GetTargetRef())
        if err != nil {
                return nil, fmt.Errorf("failed to clone target branch %s from %s: %w", ent.GetTargetRef(), ent.GetTargetCloneUrl(), err)
        }

        checkpoint := checkpoints.NewCheckpointV1Now().WithBranch(ent.GetTargetRef()).WithCommitHash(head.Hash().String())

        return &interfaces.Ingested{
                Object:     nil,
                Fs:         targetFs,
                Storer:     storer,
                BaseFs:     baseFs,
                Checkpoint: checkpoint,
        }, nil
}

func (gi *Git) fetchClone(
        ctx context.Context, url, branch string) (billy.Filesystem, storage.Storer, *plumbing.Reference, error) {
        // We clone to the memfs go-billy filesystem driver, which doesn't
        // allow for direct access to the underlying filesystem. This is
        // because we want to be able to run this in a sandboxed environment
        // where we don't have access to the underlying filesystem.
        r, err := gi.gitprov.Clone(ctx, url, branch)
        if err != nil {
                if errors.Is(err, provifv1.ErrProviderGitBranchNotFound) {
                        return nil, nil, nil, fmt.Errorf("%w: %s: branch %s", interfaces.ErrEvaluationFailed,
                                provifv1.ErrProviderGitBranchNotFound, branch)
                } else if errors.Is(err, provifv1.ErrRepositoryEmpty) {
                        return nil, nil, nil, fmt.Errorf("%w: %s", interfaces.ErrEvaluationSkipped, provifv1.ErrRepositoryEmpty)
                }
                return nil, nil, nil, err
        }

        wt, err := r.Worktree()
        if err != nil {
                return nil, nil, nil, fmt.Errorf("could not get worktree: %w", err)
        }

        head, err := r.Head()
        if err != nil {
                return nil, nil, nil, fmt.Errorf("could not get head: %w", err)
        }

        return wt.Filesystem, r.Storer, head, err
}

1	// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
2	// SPDX-License-Identifier: Apache-2.0
3
4	// Package git provides the git rule data ingest engine
5	package git
6
7	import (
8	"cmp"
9	"context"
10	"errors"
11	"fmt"
12
13	"github.com/go-git/go-billy/v5"
14	"github.com/go-git/go-git/v5/plumbing"
15	"github.com/go-git/go-git/v5/storage"
16	"github.com/go-viper/mapstructure/v2"
17	"google.golang.org/protobuf/reflect/protoreflect"
18
19	pbinternal "github.com/mindersec/minder/internal/proto"
20	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
21	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
22	"github.com/mindersec/minder/pkg/entities/v1/checkpoints"
23	provifv1 "github.com/mindersec/minder/pkg/providers/v1"
24	)
25
26	const (
27	// GitRuleDataIngestType is the type of the git rule data ingest engine
28	GitRuleDataIngestType = "git"
29	defaultBranch = "main"
30	)
31
32	// Git is the engine for a rule type that uses git data ingest
33	type Git struct {
34	cfg *pb.GitType
35	gitprov interfaces.GitProvider
36	}
37
38	// NewGitIngester creates a new git rule data ingest engine
39	func NewGitIngester(cfg pb.GitType, gitprov interfaces.GitProvider) (Git, error) {	14✔
40	if gitprov == nil {	14✔
41	return nil, fmt.Errorf("provider is nil")	×
42	}	×
43
44	if cfg == nil {	15✔
45	cfg = &pb.GitType{}	1✔
46	}	1✔
47
48	return &Git{	14✔
49	cfg: cfg,	14✔
50	gitprov: gitprov,	14✔
51	}, nil	14✔
52	}
53
54	// GetType returns the type of the git rule data ingest engine
55	func (*Git) GetType() string {	3✔
56	return GitRuleDataIngestType	3✔
57	}	3✔
58
59	// GetConfig returns the config for the git rule data ingest engine
60	func (gi *Git) GetConfig() protoreflect.ProtoMessage {	6✔
61	return gi.cfg	6✔
62	}	6✔
63
64	// Ingest does the actual data ingestion for a rule type by cloning a git repo
65	func (gi Git) Ingest(ctx context.Context, ent protoreflect.ProtoMessage, params map[string]any) (interfaces.Ingested, error) {	9✔
66	switch entity := ent.(type) {	9✔
67	case *pb.Repository:	9✔
68	return gi.ingestRepository(ctx, entity, params)	9✔
69	case *pbinternal.PullRequest:	×
70	return gi.ingestPullRequest(ctx, entity, params)	×
71	default:	×
72	return nil, fmt.Errorf("git is only supported for repositories and pull requests")	×
73	}
74	}
75
76	func (gi Git) ingestRepository(ctx context.Context, repo pb.Repository, params map[string]any) (*interfaces.Ingested, error) {	9✔
77	userCfg := &IngesterConfig{}	9✔
78	if err := mapstructure.Decode(params, userCfg); err != nil {	9✔
79	return nil, fmt.Errorf("failed to read git ingester configuration from params: %w", err)	×
80	}	×
81
82	url := cmp.Or(userCfg.CloneURL, repo.GetCloneUrl())	9✔
83	if url == "" {	9✔
84	return nil, fmt.Errorf("could not get clone url")	×
85	}	×
86
87	branch := cmp.Or(userCfg.Branch, gi.cfg.Branch, repo.GetDefaultBranch(), defaultBranch)	9✔
88	fs, storer, head, err := gi.fetchClone(ctx, url, branch)	9✔
89	if err != nil {	14✔
90	return nil, fmt.Errorf("failed to clone %s from %s: %w", branch, url, err)	5✔
91	}	5✔
92
93	hsh := head.Hash()	4✔
94		4✔
95	chkpoint := checkpoints.NewCheckpointV1Now().	4✔
96	WithBranch(branch).	4✔
97	WithCommitHash(hsh.String())	4✔
98		4✔
99	return &interfaces.Ingested{	4✔
100	Object: nil,	4✔
101	Fs: fs,	4✔
102	Storer: storer,	4✔
103	Checkpoint: chkpoint,	4✔
104	}, nil	4✔
105	}
106
107	func (gi *Git) ingestPullRequest(
108	ctx context.Context, ent pbinternal.PullRequest, params map[string]any) (interfaces.Ingested, error) {	×
109	// TODO: we don't actually have any configuration here. Do we need to read the configuration?	×
110	userCfg := &IngesterConfig{}	×
111	if err := mapstructure.Decode(params, userCfg); err != nil {	×
112	return nil, fmt.Errorf("failed to read git ingester configuration from params: %w", err)	×
113	}	×
114
115	if ent.GetBaseCloneUrl() == "" \|\| ent.GetBaseRef() == "" {	×
116	return nil, fmt.Errorf("could not get PR base branch %q from %q", ent.GetBaseRef(), ent.GetBaseCloneUrl())	×
117	}	×
118	if ent.GetTargetCloneUrl() == "" \|\| ent.GetTargetRef() == "" {	×
119	return nil, fmt.Errorf("could not get PR target branch %q from %q", ent.GetTargetRef(), ent.GetTargetCloneUrl())	×
120	}	×
121
122	baseFs, _, _, err := gi.fetchClone(ctx, ent.GetBaseCloneUrl(), ent.GetBaseRef())	×
123	if err != nil {	×
124	return nil, fmt.Errorf("failed to clone base branch %s from %s: %w", ent.GetBaseRef(), ent.GetBaseCloneUrl(), err)	×
125	}	×
126	targetFs, storer, head, err := gi.fetchClone(ctx, ent.GetTargetCloneUrl(), ent.GetTargetRef())	×
127	if err != nil {	×
128	return nil, fmt.Errorf("failed to clone target branch %s from %s: %w", ent.GetTargetRef(), ent.GetTargetCloneUrl(), err)	×
129	}	×
130
131	checkpoint := checkpoints.NewCheckpointV1Now().WithBranch(ent.GetTargetRef()).WithCommitHash(head.Hash().String())	×
132		×
133	return &interfaces.Ingested{	×
134	Object: nil,	×
135	Fs: targetFs,	×
136	Storer: storer,	×
137	BaseFs: baseFs,	×
138	Checkpoint: checkpoint,	×
139	}, nil	×
140	}
141
142	func (gi *Git) fetchClone(
143	ctx context.Context, url, branch string) (billy.Filesystem, storage.Storer, *plumbing.Reference, error) {	9✔
144	// We clone to the memfs go-billy filesystem driver, which doesn't	9✔
145	// allow for direct access to the underlying filesystem. This is	9✔
146	// because we want to be able to run this in a sandboxed environment	9✔
147	// where we don't have access to the underlying filesystem.	9✔
148	r, err := gi.gitprov.Clone(ctx, url, branch)	9✔
149	if err != nil {	14✔
150	if errors.Is(err, provifv1.ErrProviderGitBranchNotFound) {	6✔
151	return nil, nil, nil, fmt.Errorf("%w: %s: branch %s", interfaces.ErrEvaluationFailed,	1✔
152	provifv1.ErrProviderGitBranchNotFound, branch)	1✔
153	} else if errors.Is(err, provifv1.ErrRepositoryEmpty) {	5✔
NEW 154	return nil, nil, nil, fmt.Errorf("%w: %s", interfaces.ErrEvaluationSkipped, provifv1.ErrRepositoryEmpty)	×
155	}	×
156	return nil, nil, nil, err	4✔
157	}
158
159	wt, err := r.Worktree()	4✔
160	if err != nil {	4✔
161	return nil, nil, nil, fmt.Errorf("could not get worktree: %w", err)	×
162	}	×
163
164	head, err := r.Head()	4✔
165	if err != nil {	4✔
166	return nil, nil, nil, fmt.Errorf("could not get head: %w", err)	×
167	}	×
168
169	return wt.Filesystem, r.Storer, head, err	4✔
170	}

mindersec / minder / 16654340601

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous