12360611398

Committed 16 Dec 2024 08:20PM UTC coverage: 55.476% (+0.1%) from 55.374%

Build # 12360611398

Build Type

Pull #5181

github

Committed by

web-flow

Commit Message

Merge c6abe06ac into 5e3b3c802

Pull Request Pull Request #5181: Add support for base and target trees in git ingest, add .tar.gz bundler

Run Details

302 of 416 new or added lines in 10 files covered. (72.6%)

9 existing lines in 3 files now uncovered.

16963 of 30577 relevant lines covered (55.48%)

38.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

56.31

/internal/engine/ingester/git/git.go

// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
// SPDX-License-Identifier: Apache-2.0

// Package git provides the git rule data ingest engine
package git

import (
        "cmp"
        "context"
        "errors"
        "fmt"

        "github.com/go-git/go-billy/v5"
        "github.com/go-git/go-git/v5/plumbing"
        "github.com/go-git/go-git/v5/storage"
        "github.com/go-viper/mapstructure/v2"
        "google.golang.org/protobuf/reflect/protoreflect"

        engerrors "github.com/mindersec/minder/internal/engine/errors"
        pbinternal "github.com/mindersec/minder/internal/proto"
        pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
        "github.com/mindersec/minder/pkg/engine/v1/interfaces"
        "github.com/mindersec/minder/pkg/entities/v1/checkpoints"
        provifv1 "github.com/mindersec/minder/pkg/providers/v1"
)

const (
        // GitRuleDataIngestType is the type of the git rule data ingest engine
        GitRuleDataIngestType = "git"
        defaultBranch         = "main"
)

// Git is the engine for a rule type that uses git data ingest
type Git struct {
        cfg     *pb.GitType
        gitprov provifv1.Git
}

// NewGitIngester creates a new git rule data ingest engine
func NewGitIngester(cfg *pb.GitType, gitprov provifv1.Git) (*Git, error) {
        if gitprov == nil {
                return nil, fmt.Errorf("provider is nil")
        }

        if cfg == nil {
                cfg = &pb.GitType{}
        }

        return &Git{
                cfg:     cfg,
                gitprov: gitprov,
        }, nil
}

// GetType returns the type of the git rule data ingest engine
func (*Git) GetType() string {
        return GitRuleDataIngestType
}

// GetConfig returns the config for the git rule data ingest engine
func (gi *Git) GetConfig() protoreflect.ProtoMessage {
        return gi.cfg
}

// Ingest does the actual data ingestion for a rule type by cloning a git repo
func (gi *Git) Ingest(ctx context.Context, ent protoreflect.ProtoMessage, params map[string]any) (*interfaces.Result, error) {
        switch entity := ent.(type) {
        case *pb.Repository:
                return gi.ingestRepository(ctx, entity, params)
        case *pbinternal.PullRequest:
                return gi.ingestPullRequest(ctx, entity, params)
        default:
                return nil, fmt.Errorf("git is only supported for repositories and pull requests")
        }
}

func (gi *Git) ingestRepository(ctx context.Context, repo *pb.Repository, params map[string]any) (*interfaces.Result, error) {
        userCfg := &IngesterConfig{}
        if err := mapstructure.Decode(params, userCfg); err != nil {
                return nil, fmt.Errorf("failed to read git ingester configuration from params: %w", err)
        }

        url := cmp.Or(userCfg.CloneURL, repo.GetCloneUrl())
        if url == "" {
                return nil, fmt.Errorf("could not get clone url")
        }

        branch := cmp.Or(userCfg.Branch, gi.cfg.Branch, repo.GetDefaultBranch(), defaultBranch)
        fs, storer, head, err := gi.fetchClone(ctx, url, branch)
        if err != nil {
                return nil, fmt.Errorf("failed to clone %s from %s: %w", branch, url, err)
        }

        hsh := head.Hash()

        chkpoint := checkpoints.NewCheckpointV1Now().
                WithBranch(branch).
                WithCommitHash(hsh.String())

        return &interfaces.Result{
                Object:     nil,
                Fs:         fs,
                Storer:     storer,
                Checkpoint: chkpoint,
        }, nil
}

func (gi *Git) ingestPullRequest(
        ctx context.Context, ent *pbinternal.PullRequest, params map[string]any) (*interfaces.Result, error) {
        // TODO: we don't actually have any configuration here.  Do we need to read the configuration?
        userCfg := &IngesterConfig{}
        if err := mapstructure.Decode(params, userCfg); err != nil {
                return nil, fmt.Errorf("failed to read git ingester configuration from params: %w", err)
        }

        if ent.GetBaseCloneUrl() == "" || ent.GetBaseRef() == "" {
                return nil, fmt.Errorf("could not get PR base branch %q from %q", ent.GetBaseRef(), ent.GetBaseCloneUrl())
        }
        if ent.GetTargetCloneUrl() == "" || ent.GetTargetRef() == "" {
                return nil, fmt.Errorf("could not get PR target branch %q from %q", ent.GetTargetRef(), ent.GetTargetCloneUrl())
        }

        baseFs, _, _, err := gi.fetchClone(ctx, ent.GetBaseCloneUrl(), ent.GetBaseRef())
        if err != nil {
                return nil, fmt.Errorf("failed to clone base branch %s from %s: %w", ent.GetBaseRef(), ent.GetBaseCloneUrl(), err)
        }
        targetFs, storer, head, err := gi.fetchClone(ctx, ent.GetTargetCloneUrl(), ent.GetTargetRef())
        if err != nil {
                return nil, fmt.Errorf("failed to clone target branch %s from %s: %w", ent.GetTargetRef(), ent.GetTargetCloneUrl(), err)
        }

        checkpoint := checkpoints.NewCheckpointV1Now().WithBranch(ent.GetTargetRef()).WithCommitHash(head.Hash().String())

        return &interfaces.Result{
                Object:     nil,
                Fs:         targetFs,
                Storer:     storer,
                BaseFs:     baseFs,
                Checkpoint: checkpoint,
        }, nil
}

func (gi *Git) fetchClone(
        ctx context.Context, url, branch string) (billy.Filesystem, storage.Storer, *plumbing.Reference, error) {
        // We clone to the memfs go-billy filesystem driver, which doesn't
        // allow for direct access to the underlying filesystem. This is
        // because we want to be able to run this in a sandboxed environment
        // where we don't have access to the underlying filesystem.
        r, err := gi.gitprov.Clone(ctx, url, branch)
        if err != nil {
                if errors.Is(err, provifv1.ErrProviderGitBranchNotFound) {
                        return nil, nil, nil, fmt.Errorf("%w: %s: branch %s", engerrors.ErrEvaluationFailed,
                                provifv1.ErrProviderGitBranchNotFound, branch)
                } else if errors.Is(err, provifv1.ErrRepositoryEmpty) {
                        return nil, nil, nil, fmt.Errorf("%w: %s", engerrors.ErrEvaluationSkipped, provifv1.ErrRepositoryEmpty)
                }
                return nil, nil, nil, err
        }

        wt, err := r.Worktree()
        if err != nil {
                return nil, nil, nil, fmt.Errorf("could not get worktree: %w", err)
        }

        head, err := r.Head()
        if err != nil {
                return nil, nil, nil, fmt.Errorf("could not get head: %w", err)
        }

        return wt.Filesystem, r.Storer, head, err
}

1	// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
2	// SPDX-License-Identifier: Apache-2.0
3
4	// Package git provides the git rule data ingest engine
5	package git
6
7	import (
8	"cmp"
9	"context"
10	"errors"
11	"fmt"
12
13	"github.com/go-git/go-billy/v5"
14	"github.com/go-git/go-git/v5/plumbing"
15	"github.com/go-git/go-git/v5/storage"
16	"github.com/go-viper/mapstructure/v2"
17	"google.golang.org/protobuf/reflect/protoreflect"
18
19	engerrors "github.com/mindersec/minder/internal/engine/errors"
20	pbinternal "github.com/mindersec/minder/internal/proto"
21	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
22	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
23	"github.com/mindersec/minder/pkg/entities/v1/checkpoints"
24	provifv1 "github.com/mindersec/minder/pkg/providers/v1"
25	)
26
27	const (
28	// GitRuleDataIngestType is the type of the git rule data ingest engine
29	GitRuleDataIngestType = "git"
30	defaultBranch = "main"
31	)
32
33	// Git is the engine for a rule type that uses git data ingest
34	type Git struct {
35	cfg *pb.GitType
36	gitprov provifv1.Git
37	}
38
39	// NewGitIngester creates a new git rule data ingest engine
40	func NewGitIngester(cfg pb.GitType, gitprov provifv1.Git) (Git, error) {	14✔
41	if gitprov == nil {	14✔
42	return nil, fmt.Errorf("provider is nil")	×
43	}	×
44
45	if cfg == nil {	15✔
46	cfg = &pb.GitType{}	1✔
47	}	1✔
48
49	return &Git{	14✔
50	cfg: cfg,	14✔
51	gitprov: gitprov,	14✔
52	}, nil	14✔
53	}
54
55	// GetType returns the type of the git rule data ingest engine
56	func (*Git) GetType() string {	3✔
57	return GitRuleDataIngestType	3✔
58	}	3✔
59
60	// GetConfig returns the config for the git rule data ingest engine
61	func (gi *Git) GetConfig() protoreflect.ProtoMessage {	6✔
62	return gi.cfg	6✔
63	}	6✔
64
65	// Ingest does the actual data ingestion for a rule type by cloning a git repo
66	func (gi Git) Ingest(ctx context.Context, ent protoreflect.ProtoMessage, params map[string]any) (interfaces.Result, error) {	9✔
67	switch entity := ent.(type) {	9✔
68	case *pb.Repository:	9✔
69	return gi.ingestRepository(ctx, entity, params)	9✔
NEW 70	case *pbinternal.PullRequest:	×
NEW 71	return gi.ingestPullRequest(ctx, entity, params)	×
NEW 72	default:	×
NEW 73	return nil, fmt.Errorf("git is only supported for repositories and pull requests")	×
74	}
75	}
76
77	func (gi Git) ingestRepository(ctx context.Context, repo pb.Repository, params map[string]any) (*interfaces.Result, error) {	9✔
78	userCfg := &IngesterConfig{}	9✔
79	if err := mapstructure.Decode(params, userCfg); err != nil {	9✔
80	return nil, fmt.Errorf("failed to read git ingester configuration from params: %w", err)	×
81	}	×
82
83	url := cmp.Or(userCfg.CloneURL, repo.GetCloneUrl())	9✔
84	if url == "" {	9✔
85	return nil, fmt.Errorf("could not get clone url")	×
86	}	×
87
88	branch := cmp.Or(userCfg.Branch, gi.cfg.Branch, repo.GetDefaultBranch(), defaultBranch)	9✔
89	fs, storer, head, err := gi.fetchClone(ctx, url, branch)	9✔
90	if err != nil {	14✔
91	return nil, fmt.Errorf("failed to clone %s from %s: %w", branch, url, err)	5✔
92	}	5✔
93
94	hsh := head.Hash()	4✔
95		4✔
96	chkpoint := checkpoints.NewCheckpointV1Now().	4✔
97	WithBranch(branch).	4✔
98	WithCommitHash(hsh.String())	4✔
99		4✔
100	return &interfaces.Result{	4✔
101	Object: nil,	4✔
102	Fs: fs,	4✔
103	Storer: storer,	4✔
104	Checkpoint: chkpoint,	4✔
105	}, nil	4✔
106	}
107
108	func (gi *Git) ingestPullRequest(
NEW 109	ctx context.Context, ent pbinternal.PullRequest, params map[string]any) (interfaces.Result, error) {	×
NEW 110	// TODO: we don't actually have any configuration here. Do we need to read the configuration?	×
NEW 111	userCfg := &IngesterConfig{}	×
NEW 112	if err := mapstructure.Decode(params, userCfg); err != nil {	×
NEW 113	return nil, fmt.Errorf("failed to read git ingester configuration from params: %w", err)	×
UNCOV 114	}	×
115
NEW 116	if ent.GetBaseCloneUrl() == "" \|\| ent.GetBaseRef() == "" {	×
NEW 117	return nil, fmt.Errorf("could not get PR base branch %q from %q", ent.GetBaseRef(), ent.GetBaseCloneUrl())	×
NEW 118	}	×
NEW 119	if ent.GetTargetCloneUrl() == "" \|\| ent.GetTargetRef() == "" {	×
NEW 120	return nil, fmt.Errorf("could not get PR target branch %q from %q", ent.GetTargetRef(), ent.GetTargetCloneUrl())	×
UNCOV 121	}	×
122
NEW 123	baseFs, _, _, err := gi.fetchClone(ctx, ent.GetBaseCloneUrl(), ent.GetBaseRef())	×
NEW 124	if err != nil {	×
NEW 125	return nil, fmt.Errorf("failed to clone base branch %s from %s: %w", ent.GetBaseRef(), ent.GetBaseCloneUrl(), err)	×
NEW 126	}	×
NEW 127	targetFs, storer, head, err := gi.fetchClone(ctx, ent.GetTargetCloneUrl(), ent.GetTargetRef())	×
NEW 128	if err != nil {	×
NEW 129	return nil, fmt.Errorf("failed to clone target branch %s from %s: %w", ent.GetTargetRef(), ent.GetTargetCloneUrl(), err)	×
UNCOV 130	}	×
131
NEW 132	checkpoint := checkpoints.NewCheckpointV1Now().WithBranch(ent.GetTargetRef()).WithCommitHash(head.Hash().String())	×
NEW 133		×
NEW 134	return &interfaces.Result{	×
NEW 135	Object: nil,	×
NEW 136	Fs: targetFs,	×
NEW 137	Storer: storer,	×
NEW 138	BaseFs: baseFs,	×
NEW 139	Checkpoint: checkpoint,	×
NEW 140	}, nil	×
141	}
142
143	func (gi *Git) fetchClone(
144	ctx context.Context, url, branch string) (billy.Filesystem, storage.Storer, *plumbing.Reference, error) {	9✔
145	// We clone to the memfs go-billy filesystem driver, which doesn't	9✔
146	// allow for direct access to the underlying filesystem. This is	9✔
147	// because we want to be able to run this in a sandboxed environment	9✔
148	// where we don't have access to the underlying filesystem.	9✔
149	r, err := gi.gitprov.Clone(ctx, url, branch)	9✔
150	if err != nil {	14✔
151	if errors.Is(err, provifv1.ErrProviderGitBranchNotFound) {	6✔
152	return nil, nil, nil, fmt.Errorf("%w: %s: branch %s", engerrors.ErrEvaluationFailed,	1✔
153	provifv1.ErrProviderGitBranchNotFound, branch)	1✔
154	} else if errors.Is(err, provifv1.ErrRepositoryEmpty) {	5✔
NEW 155	return nil, nil, nil, fmt.Errorf("%w: %s", engerrors.ErrEvaluationSkipped, provifv1.ErrRepositoryEmpty)	×
NEW 156	}	×
157	return nil, nil, nil, err	4✔
158	}
159
160	wt, err := r.Worktree()	4✔
161	if err != nil {	4✔
NEW 162	return nil, nil, nil, fmt.Errorf("could not get worktree: %w", err)	×
UNCOV 163	}	×
164
165	head, err := r.Head()	4✔
166	if err != nil {	4✔
NEW 167	return nil, nil, nil, fmt.Errorf("could not get head: %w", err)	×
UNCOV 168	}	×
169
170	return wt.Filesystem, r.Storer, head, err	4✔
171	}

mindersec / minder / 12360611398

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous