15981216547

Committed 30 Jun 2025 06:52PM UTC coverage: 57.393% (+0.02%) from 57.377%

Build # 15981216547

Build Type

Pull #5702

github

Committed by

web-flow

Commit Message

Merge b1a7618c3 into 9b4f171a0

Pull Request Pull Request #5702: Add templates for REST and YQ eval

Run Details

57 of 78 new or added lines in 7 files covered. (73.08%)

21 existing lines in 3 files now uncovered.

18593 of 32396 relevant lines covered (57.39%)

37.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

48.36

/internal/engine/ingester/diff/diff.go

// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
// SPDX-License-Identifier: Apache-2.0

// Package diff provides the diff rule data ingest engine
package diff

import (
        "bufio"
        "cmp"
        "context"
        "fmt"
        "math"
        "path/filepath"
        "regexp"
        "slices"
        "strconv"
        "strings"

        "github.com/go-git/go-billy/v5"
        "github.com/go-git/go-billy/v5/helper/iofs"
        scalibr "github.com/google/osv-scalibr"
        "github.com/google/osv-scalibr/extractor"
        "github.com/google/osv-scalibr/extractor/filesystem/list"
        scalibr_fs "github.com/google/osv-scalibr/fs"
        scalibr_plugin "github.com/google/osv-scalibr/plugin"
        "github.com/google/osv-scalibr/purl"
        "github.com/rs/zerolog"
        "google.golang.org/protobuf/reflect/protoreflect"

        pbinternal "github.com/mindersec/minder/internal/proto"
        pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
        "github.com/mindersec/minder/pkg/engine/v1/interfaces"
        "github.com/mindersec/minder/pkg/entities/v1/checkpoints"
)

const (
        // DiffRuleDataIngestType is the type of the diff rule data ingest engine
        DiffRuleDataIngestType = "diff"
        prFilesPerPage         = 30
        wildcard               = "*"
)

// Diff is the diff rule data ingest engine
type Diff struct {
        cli interfaces.GitHubListAndClone
        cfg *pb.DiffType
}

// NewDiffIngester creates a new diff ingester
func NewDiffIngester(
        cfg *pb.DiffType,
        cli interfaces.GitHubListAndClone,
) (*Diff, error) {
        if cfg == nil {
                cfg = &pb.DiffType{}
        }

        if cli == nil {
                return nil, fmt.Errorf("provider is nil")
        }

        return &Diff{
                cfg: cfg,
                cli: cli,
        }, nil
}

// GetType returns the type of the diff rule data ingest engine
func (*Diff) GetType() string {
        return DiffRuleDataIngestType
}

// GetConfig returns the config for the diff rule data ingest engine
func (di *Diff) GetConfig() protoreflect.ProtoMessage {
        return di.cfg
}

// Ingest ingests a diff from a pull request in accordance with its type
func (di *Diff) Ingest(
        ctx context.Context,
        ent protoreflect.ProtoMessage,
        _ map[string]any,
) (*interfaces.Ingested, error) {
        pr, ok := ent.(*pbinternal.PullRequest)
        if !ok {
                return nil, fmt.Errorf("entity is not a pull request")
        }

        // The GitHub Go API takes an int32, but our proto stores an int64; make sure we don't overflow
        if pr.Number > math.MaxInt {
                return nil, fmt.Errorf("pr number is too large")
        }
        prNumber := int(pr.Number)

        switch di.cfg.GetType() {
        case "", pb.DiffTypeDep:
                return di.getDepTypeDiff(ctx, prNumber, pr)

        case pb.DiffTypeNewDeps:
                // TODO: once we've tested some, convert DiffTypeDep to use this algorithm.
                return di.getScalibrTypeDiff(ctx, prNumber, pr)

        case pb.DiffTypeFull:
                return di.getFullTypeDiff(ctx, prNumber, pr)

        default:
                return nil, fmt.Errorf("unknown diff type")
        }
}

func (di *Diff) getDepTypeDiff(ctx context.Context, prNumber int, pr *pbinternal.PullRequest) (*interfaces.Ingested, error) {
        deps := pbinternal.PrDependencies{Pr: pr}
        page := 0

        for {
                prFiles, resp, err := di.cli.ListFiles(ctx, pr.RepoOwner, pr.RepoName, prNumber, prFilesPerPage, page)
                if err != nil {
                        return nil, fmt.Errorf("error getting pull request files: %w", err)
                }

                for _, file := range prFiles {
                        fileDiffs, err := di.ingestFileForDepDiff(file.GetFilename(), file.GetPatch(), file.GetRawURL(), *zerolog.Ctx(ctx))
                        if err != nil {
                                return nil, fmt.Errorf("error ingesting file %s: %w", file.GetFilename(), err)
                        }
                        deps.Deps = append(deps.Deps, fileDiffs...)
                }

                if resp.NextPage == 0 {
                        break
                }

                page = resp.NextPage
        }

        return &interfaces.Ingested{Object: &deps, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil
}

func (di *Diff) getFullTypeDiff(ctx context.Context, prNumber int, pr *pbinternal.PullRequest) (*interfaces.Ingested, error) {
        diff := &pbinternal.PrContents{Pr: pr}
        page := 0

        for {
                prFiles, resp, err := di.cli.ListFiles(ctx, pr.RepoOwner, pr.RepoName, prNumber, prFilesPerPage, page)
                if err != nil {
                        return nil, fmt.Errorf("error getting pull request files: %w", err)
                }

                for _, file := range prFiles {
                        fileDiffs, err := ingestFileForFullDiff(file.GetFilename(), file.GetPatch(), file.GetRawURL())
                        if err != nil {
                                return nil, fmt.Errorf("error ingesting file %s: %w", file.GetFilename(), err)
                        }
                        diff.Files = append(diff.Files, fileDiffs)
                }

                if resp.NextPage == 0 {
                        break
                }

                page = resp.NextPage
        }

        return &interfaces.Ingested{Object: diff, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil
}

func (di *Diff) ingestFileForDepDiff(
        filename, patchContents, patchUrl string,
        logger zerolog.Logger,
) ([]*pbinternal.PrDependencies_ContextualDependency, error) {
        parser := di.getParserForFile(filename, logger)
        if parser == nil {
                return nil, nil
        }

        depBatch, err := parser(patchContents)
        if err != nil {
                return nil, fmt.Errorf("error parsing file %s: %w", filename, err)
        }

        batchCtxDeps := make([]*pbinternal.PrDependencies_ContextualDependency, 0, len(depBatch))
        for i := range depBatch {
                dep := depBatch[i]
                batchCtxDeps = append(batchCtxDeps, &pbinternal.PrDependencies_ContextualDependency{
                        Dep: dep,
                        File: &pbinternal.PrDependencies_ContextualDependency_FilePatch{
                                Name:     filename,
                                PatchUrl: patchUrl,
                        },
                })
        }

        return batchCtxDeps, nil
}

func (di *Diff) getScalibrTypeDiff(ctx context.Context, _ int, pr *pbinternal.PullRequest) (*interfaces.Ingested, error) {
        deps := pbinternal.PrDependencies{Pr: pr}

        // TODO: we should be able to just fetch the additional commits between base and target.
        // Our current Git abstraction isn't quite powerful enough, so we do two shallow clones.

        baseInventory, err := di.scalibrInventory(ctx, pr.BaseCloneUrl, pr.BaseRef)
        if err != nil {
                return nil, fmt.Errorf("failed to clone base from %s at %q: %w", pr.BaseCloneUrl, pr.BaseRef, err)
        }
        newInventory, err := di.scalibrInventory(ctx, pr.TargetCloneUrl, pr.TargetRef)
        if err != nil {
                return nil, fmt.Errorf("failed to clone fork from %s at %q: %w", pr.TargetCloneUrl, pr.TargetRef, err)
        }

        newDeps := setDifference(baseInventory, newInventory, inventorySorter)

        deps.Deps = make([]*pbinternal.PrDependencies_ContextualDependency, 0, len(newDeps))
        for _, inventory := range newDeps {
                for _, filename := range inventory.Locations {
                        deps.Deps = append(deps.Deps, &pbinternal.PrDependencies_ContextualDependency{
                                Dep: &pbinternal.Dependency{
                                        Ecosystem: inventoryToEcosystem(inventory),
                                        Name:      inventory.Name,
                                        Version:   inventory.Version,
                                },
                                File: &pbinternal.PrDependencies_ContextualDependency_FilePatch{
                                        Name:     filename,
                                        PatchUrl: "", // TODO: do we need this?
                                },
                        })
                }
        }

        return &interfaces.Ingested{Object: &deps, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil
}

func inventorySorter(a *extractor.Package, b *extractor.Package) int {
        // If we compare by name and version first, we can avoid serializing Locations to strings
        res := cmp.Or(cmp.Compare(a.Name, b.Name), cmp.Compare(a.Version, b.Version))
        if res != 0 {
                return res
        }
        // TODO: Locations should probably be sorted, but scalibr is going to export a compare function.
        aLoc := fmt.Sprintf("%v", a.Locations)
        bLoc := fmt.Sprintf("%v", b.Locations)
        return cmp.Compare(aLoc, bLoc)
}

func (di *Diff) scalibrInventory(ctx context.Context, repoURL string, ref string) ([]*extractor.Package, error) {
        clone, err := di.cli.Clone(ctx, repoURL, ref)
        if err != nil {
                return nil, err
        }

        tree, err := clone.Worktree()
        if err != nil {
                return nil, err
        }
        return scanFs(ctx, tree.Filesystem, map[string]string{})
}

func scanFs(ctx context.Context, memFS billy.Filesystem, _ map[string]string) ([]*extractor.Package, error) {
        // have to down-cast here, because scalibr needs multiple io/fs types
        wrapped, ok := iofs.New(memFS).(scalibr_fs.FS)
        if !ok {
                return nil, fmt.Errorf("error converting filesystem to ReadDirFS")
        }

        desiredCaps := scalibr_plugin.Capabilities{
                OS:            scalibr_plugin.OSLinux,
                Network:       scalibr_plugin.NetworkOffline,
                DirectFS:      false,
                RunningSystem: false,
        }

        scalibrFs := scalibr_fs.ScanRoot{FS: wrapped}
        scanConfig := scalibr.ScanConfig{
                ScanRoots: []*scalibr_fs.ScanRoot{&scalibrFs},
                // All includes Ruby, Dotnet which we're not ready to test yet, so use the more limited Default set.
                FilesystemExtractors: list.FromCapabilities(&desiredCaps),
                Capabilities:         &desiredCaps,
        }

        scanner := scalibr.New()
        scanResults := scanner.Scan(ctx, &scanConfig)

        if scanResults == nil || scanResults.Status == nil {
                return nil, fmt.Errorf("error scanning files: no results")
        }
        if scanResults.Status.Status != scalibr_plugin.ScanStatusSucceeded {
                return nil, fmt.Errorf("error scanning files: %s", scanResults.Status)
        }

        return scanResults.Inventory.Packages, nil
}

func inventoryToEcosystem(inventory *extractor.Package) pbinternal.DepEcosystem {
        if inventory == nil {
                zerolog.Ctx(context.Background()).Warn().Msg("nil ecosystem scanning diffs")
                return pbinternal.DepEcosystem_DEP_ECOSYSTEM_UNSPECIFIED
        }

        package_url := inventory.PURL()

        // Sometimes Scalibr uses the string "PyPI" instead of "pypi" when reporting the ecosystem.
        switch package_url.Type {
        // N.B. using an enum here abitrarily restricts our ability to add new
        // ecosystems without a core minder change.  Switching to strings ala
        // purl might be an improvement.
        case purl.TypePyPi:
                return pbinternal.DepEcosystem_DEP_ECOSYSTEM_PYPI
        case purl.TypeNPM:
                return pbinternal.DepEcosystem_DEP_ECOSYSTEM_NPM
        case purl.TypeGolang:
                return pbinternal.DepEcosystem_DEP_ECOSYSTEM_GO
        default:
                return pbinternal.DepEcosystem_DEP_ECOSYSTEM_UNSPECIFIED
        }
}

// ingestFileForFullDiff processes a given file's patch from a pull request.
// It scans through the patch line by line, identifying the changes made.
// If it's a hunk header, it extracts the starting line number. If it's an addition, it records the line content and its number.
// The function also increments the line number for context lines (lines that provide context but haven't been modified).
func ingestFileForFullDiff(filename, patch, patchUrl string) (*pbinternal.PrContents_File, error) {
        var result []*pbinternal.PrContents_File_Line

        scanner := bufio.NewScanner(strings.NewReader(patch))
        regex := regexp.MustCompile(`@@ -\d+,\d+ \+(\d+),\d+ @@`)

        var currentLineNumber int64
        var err error
        for scanner.Scan() {
                line := scanner.Text()

                if matches := regex.FindStringSubmatch(line); matches != nil {
                        currentLineNumber, err = strconv.ParseInt(matches[1], 10, 32)
                        if err != nil {
                                return nil, fmt.Errorf("error parsing line number from the hunk header: %w", err)
                        }
                } else if strings.HasPrefix(line, "+") {
                        result = append(result, &pbinternal.PrContents_File_Line{
                                Content: line[1:],
                                // see the use of strconv.ParseInt above: this is a safe downcast
                                // nolint: gosec
                                LineNumber: int32(currentLineNumber),
                        })

                        currentLineNumber++
                } else if !strings.HasPrefix(line, "-") {
                        currentLineNumber++
                }
        }

        if err := scanner.Err(); err != nil {
                return nil, fmt.Errorf("error reading patch: %w", err)
        }

        return &pbinternal.PrContents_File{
                Name:         filename,
                FilePatchUrl: patchUrl,
                PatchLines:   result,
        }, nil
}

func (di *Diff) getEcosystemForFile(filename string) DependencyEcosystem {
        lastComponent := filepath.Base(filename)

        for _, ecoMapping := range di.cfg.Ecosystems {
                if match, _ := filepath.Match(ecoMapping.Depfile, lastComponent); match {
                        return DependencyEcosystem(ecoMapping.Name)
                }
        }
        return DepEcosystemNone
}

func (di *Diff) getParserForFile(filename string, logger zerolog.Logger) ecosystemParser {
        eco := di.getEcosystemForFile(filename)
        if eco == DepEcosystemNone {
                logger.Debug().
                        Str("filename", filename).
                        Msg("No ecosystem found, skipping")
                return nil
        }

        logger.Debug().
                Str("filename", filename).
                Str("package-ecosystem", string(eco)).
                Msg("matched ecosystem")

        return newEcosystemParser(eco)
}

// Computes the set of elements in updated which are not in base.
// Note: this function may permute (sort) the order of elements in base and updated.
func setDifference[Slice ~[]E, E any](base Slice, updated Slice, sorter func(a, b E) int) Slice {

        slices.SortFunc(base, sorter)
        slices.SortFunc(updated, sorter)

        baseIdx, newIdx := 0, 0
        ret := make(Slice, 0)
        for baseIdx < len(base) && newIdx < len(updated) {
                cmpResult := sorter(base[baseIdx], updated[newIdx])
                if cmpResult < 0 {
                        baseIdx++
                } else if cmpResult > 0 {
                        ret = append(ret, updated[newIdx])
                        newIdx++
                } else {
                        baseIdx++
                        newIdx++
                }
        }
        if newIdx < len(updated) {
                ret = append(ret, updated[newIdx:]...)
        }

        // TODO: add metric for number of deps scanned vs total deps

        return ret
}

1	// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
2	// SPDX-License-Identifier: Apache-2.0
3
4	// Package diff provides the diff rule data ingest engine
5	package diff
6
7	import (
8	"bufio"
9	"cmp"
10	"context"
11	"fmt"
12	"math"
13	"path/filepath"
14	"regexp"
15	"slices"
16	"strconv"
17	"strings"
18
19	"github.com/go-git/go-billy/v5"
20	"github.com/go-git/go-billy/v5/helper/iofs"
21	scalibr "github.com/google/osv-scalibr"
22	"github.com/google/osv-scalibr/extractor"
23	"github.com/google/osv-scalibr/extractor/filesystem/list"
24	scalibr_fs "github.com/google/osv-scalibr/fs"
25	scalibr_plugin "github.com/google/osv-scalibr/plugin"
26	"github.com/google/osv-scalibr/purl"
27	"github.com/rs/zerolog"
28	"google.golang.org/protobuf/reflect/protoreflect"
29
30	pbinternal "github.com/mindersec/minder/internal/proto"
31	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
32	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
33	"github.com/mindersec/minder/pkg/entities/v1/checkpoints"
34	)
35
36	const (
37	// DiffRuleDataIngestType is the type of the diff rule data ingest engine
38	DiffRuleDataIngestType = "diff"
39	prFilesPerPage = 30
40	wildcard = "*"
41	)
42
43	// Diff is the diff rule data ingest engine
44	type Diff struct {
45	cli interfaces.GitHubListAndClone
46	cfg *pb.DiffType
47	}
48
49	// NewDiffIngester creates a new diff ingester
50	func NewDiffIngester(
51	cfg *pb.DiffType,
52	cli interfaces.GitHubListAndClone,
53	) (*Diff, error) {	4✔
54	if cfg == nil {	4✔
55	cfg = &pb.DiffType{}	×
56	}	×
57
58	if cli == nil {	4✔
59	return nil, fmt.Errorf("provider is nil")	×
60	}	×
61
62	return &Diff{	4✔
63	cfg: cfg,	4✔
64	cli: cli,	4✔
65	}, nil	4✔
66	}
67
68	// GetType returns the type of the diff rule data ingest engine
69	func (*Diff) GetType() string {	3✔
70	return DiffRuleDataIngestType	3✔
71	}	3✔
72
73	// GetConfig returns the config for the diff rule data ingest engine
74	func (di *Diff) GetConfig() protoreflect.ProtoMessage {	6✔
75	return di.cfg	6✔
76	}	6✔
77
78	// Ingest ingests a diff from a pull request in accordance with its type
79	func (di *Diff) Ingest(
80	ctx context.Context,
81	ent protoreflect.ProtoMessage,
82	_ map[string]any,
83	) (*interfaces.Ingested, error) {	4✔
84	pr, ok := ent.(*pbinternal.PullRequest)	4✔
85	if !ok {	4✔
86	return nil, fmt.Errorf("entity is not a pull request")	×
87	}	×
88
89	// The GitHub Go API takes an int32, but our proto stores an int64; make sure we don't overflow
90	if pr.Number > math.MaxInt {	4✔
91	return nil, fmt.Errorf("pr number is too large")	×
92	}	×
93	prNumber := int(pr.Number)	4✔
94		4✔
95	switch di.cfg.GetType() {	4✔
96	case "", pb.DiffTypeDep:	×
97	return di.getDepTypeDiff(ctx, prNumber, pr)	×
98
99	case pb.DiffTypeNewDeps:	4✔
100	// TODO: once we've tested some, convert DiffTypeDep to use this algorithm.	4✔
101	return di.getScalibrTypeDiff(ctx, prNumber, pr)	4✔
102
103	case pb.DiffTypeFull:	×
104	return di.getFullTypeDiff(ctx, prNumber, pr)	×
105
106	default:	×
107	return nil, fmt.Errorf("unknown diff type")	×
108	}
109	}
110
111	func (di Diff) getDepTypeDiff(ctx context.Context, prNumber int, pr pbinternal.PullRequest) (*interfaces.Ingested, error) {	×
112	deps := pbinternal.PrDependencies{Pr: pr}	×
113	page := 0	×
114		×
115	for {	×
116	prFiles, resp, err := di.cli.ListFiles(ctx, pr.RepoOwner, pr.RepoName, prNumber, prFilesPerPage, page)	×
117	if err != nil {	×
118	return nil, fmt.Errorf("error getting pull request files: %w", err)	×
119	}	×
120
121	for _, file := range prFiles {	×
122	fileDiffs, err := di.ingestFileForDepDiff(file.GetFilename(), file.GetPatch(), file.GetRawURL(), *zerolog.Ctx(ctx))	×
123	if err != nil {	×
124	return nil, fmt.Errorf("error ingesting file %s: %w", file.GetFilename(), err)	×
125	}	×
126	deps.Deps = append(deps.Deps, fileDiffs...)	×
127	}
128
129	if resp.NextPage == 0 {	×
130	break	×
131	}
132
133	page = resp.NextPage	×
134	}
135
136	return &interfaces.Ingested{Object: &deps, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil	×
137	}
138
139	func (di Diff) getFullTypeDiff(ctx context.Context, prNumber int, pr pbinternal.PullRequest) (*interfaces.Ingested, error) {	×
140	diff := &pbinternal.PrContents{Pr: pr}	×
141	page := 0	×
142		×
143	for {	×
144	prFiles, resp, err := di.cli.ListFiles(ctx, pr.RepoOwner, pr.RepoName, prNumber, prFilesPerPage, page)	×
145	if err != nil {	×
146	return nil, fmt.Errorf("error getting pull request files: %w", err)	×
147	}	×
148
149	for _, file := range prFiles {	×
150	fileDiffs, err := ingestFileForFullDiff(file.GetFilename(), file.GetPatch(), file.GetRawURL())	×
151	if err != nil {	×
152	return nil, fmt.Errorf("error ingesting file %s: %w", file.GetFilename(), err)	×
153	}	×
154	diff.Files = append(diff.Files, fileDiffs)	×
155	}
156
157	if resp.NextPage == 0 {	×
158	break	×
159	}
160
161	page = resp.NextPage	×
162	}
163
164	return &interfaces.Ingested{Object: diff, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil	×
165	}
166
167	func (di *Diff) ingestFileForDepDiff(
168	filename, patchContents, patchUrl string,
169	logger zerolog.Logger,
170	) ([]*pbinternal.PrDependencies_ContextualDependency, error) {	×
171	parser := di.getParserForFile(filename, logger)	×
172	if parser == nil {	×
173	return nil, nil	×
174	}	×
175
176	depBatch, err := parser(patchContents)	×
177	if err != nil {	×
178	return nil, fmt.Errorf("error parsing file %s: %w", filename, err)	×
179	}	×
180
181	batchCtxDeps := make([]*pbinternal.PrDependencies_ContextualDependency, 0, len(depBatch))	×
182	for i := range depBatch {	×
183	dep := depBatch[i]	×
184	batchCtxDeps = append(batchCtxDeps, &pbinternal.PrDependencies_ContextualDependency{	×
185	Dep: dep,	×
186	File: &pbinternal.PrDependencies_ContextualDependency_FilePatch{	×
187	Name: filename,	×
188	PatchUrl: patchUrl,	×
189	},	×
190	})	×
191	}	×
192
193	return batchCtxDeps, nil	×
194	}
195
196	func (di Diff) getScalibrTypeDiff(ctx context.Context, _ int, pr pbinternal.PullRequest) (*interfaces.Ingested, error) {	4✔
197	deps := pbinternal.PrDependencies{Pr: pr}	4✔
198		4✔
199	// TODO: we should be able to just fetch the additional commits between base and target.	4✔
200	// Our current Git abstraction isn't quite powerful enough, so we do two shallow clones.	4✔
201		4✔
202	baseInventory, err := di.scalibrInventory(ctx, pr.BaseCloneUrl, pr.BaseRef)	4✔
203	if err != nil {	4✔
204	return nil, fmt.Errorf("failed to clone base from %s at %q: %w", pr.BaseCloneUrl, pr.BaseRef, err)	×
205	}	×
206	newInventory, err := di.scalibrInventory(ctx, pr.TargetCloneUrl, pr.TargetRef)	4✔
207	if err != nil {	4✔
208	return nil, fmt.Errorf("failed to clone fork from %s at %q: %w", pr.TargetCloneUrl, pr.TargetRef, err)	×
209	}	×
210
211	newDeps := setDifference(baseInventory, newInventory, inventorySorter)	4✔
212		4✔
213	deps.Deps = make([]*pbinternal.PrDependencies_ContextualDependency, 0, len(newDeps))	4✔
214	for _, inventory := range newDeps {	10✔
215	for _, filename := range inventory.Locations {	12✔
216	deps.Deps = append(deps.Deps, &pbinternal.PrDependencies_ContextualDependency{	6✔
217	Dep: &pbinternal.Dependency{	6✔
218	Ecosystem: inventoryToEcosystem(inventory),	6✔
219	Name: inventory.Name,	6✔
220	Version: inventory.Version,	6✔
221	},	6✔
222	File: &pbinternal.PrDependencies_ContextualDependency_FilePatch{	6✔
223	Name: filename,	6✔
224	PatchUrl: "", // TODO: do we need this?	6✔
225	},	6✔
226	})	6✔
227	}	6✔
228	}
229
230	return &interfaces.Ingested{Object: &deps, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil	4✔
231	}
232
233	func inventorySorter(a extractor.Package, b extractor.Package) int {	22✔
234	// If we compare by name and version first, we can avoid serializing Locations to strings	22✔
235	res := cmp.Or(cmp.Compare(a.Name, b.Name), cmp.Compare(a.Version, b.Version))	22✔
236	if res != 0 {	42✔
237	return res	20✔
238	}	20✔
239	// TODO: Locations should probably be sorted, but scalibr is going to export a compare function.
240	aLoc := fmt.Sprintf("%v", a.Locations)	2✔
241	bLoc := fmt.Sprintf("%v", b.Locations)	2✔
242	return cmp.Compare(aLoc, bLoc)	2✔
243	}
244
245	func (di Diff) scalibrInventory(ctx context.Context, repoURL string, ref string) ([]extractor.Package, error) {	8✔
246	clone, err := di.cli.Clone(ctx, repoURL, ref)	8✔
247	if err != nil {	8✔
248	return nil, err	×
249	}	×
250
251	tree, err := clone.Worktree()	8✔
252	if err != nil {	8✔
253	return nil, err	×
254	}	×
255	return scanFs(ctx, tree.Filesystem, map[string]string{})	8✔
256	}
257
258	func scanFs(ctx context.Context, memFS billy.Filesystem, _ map[string]string) ([]*extractor.Package, error) {	8✔
259	// have to down-cast here, because scalibr needs multiple io/fs types	8✔
260	wrapped, ok := iofs.New(memFS).(scalibr_fs.FS)	8✔
261	if !ok {	8✔
262	return nil, fmt.Errorf("error converting filesystem to ReadDirFS")	×
263	}	×
264
265	desiredCaps := scalibr_plugin.Capabilities{	8✔
266	OS: scalibr_plugin.OSLinux,	8✔
267	Network: scalibr_plugin.NetworkOffline,	8✔
268	DirectFS: false,	8✔
269	RunningSystem: false,	8✔
270	}	8✔
271		8✔
272	scalibrFs := scalibr_fs.ScanRoot{FS: wrapped}	8✔
273	scanConfig := scalibr.ScanConfig{	8✔
274	ScanRoots: []*scalibr_fs.ScanRoot{&scalibrFs},	8✔
275	// All includes Ruby, Dotnet which we're not ready to test yet, so use the more limited Default set.	8✔
276	FilesystemExtractors: list.FromCapabilities(&desiredCaps),	8✔
277	Capabilities: &desiredCaps,	8✔
278	}	8✔
279		8✔
280	scanner := scalibr.New()	8✔
281	scanResults := scanner.Scan(ctx, &scanConfig)	8✔
282		8✔
283	if scanResults == nil \|\| scanResults.Status == nil {	8✔
284	return nil, fmt.Errorf("error scanning files: no results")	×
285	}	×
286	if scanResults.Status.Status != scalibr_plugin.ScanStatusSucceeded {	8✔
287	return nil, fmt.Errorf("error scanning files: %s", scanResults.Status)	×
288	}	×
289
290	return scanResults.Inventory.Packages, nil	8✔
291	}
292
293	func inventoryToEcosystem(inventory *extractor.Package) pbinternal.DepEcosystem {	6✔
294	if inventory == nil {	6✔
295	zerolog.Ctx(context.Background()).Warn().Msg("nil ecosystem scanning diffs")	×
296	return pbinternal.DepEcosystem_DEP_ECOSYSTEM_UNSPECIFIED	×
297	}	×
298
299	package_url := inventory.PURL()	6✔
300		6✔
301	// Sometimes Scalibr uses the string "PyPI" instead of "pypi" when reporting the ecosystem.	6✔
302	switch package_url.Type {	6✔
303	// N.B. using an enum here abitrarily restricts our ability to add new
304	// ecosystems without a core minder change. Switching to strings ala
305	// purl might be an improvement.
306	case purl.TypePyPi:	2✔
307	return pbinternal.DepEcosystem_DEP_ECOSYSTEM_PYPI	2✔
308	case purl.TypeNPM:	2✔
309	return pbinternal.DepEcosystem_DEP_ECOSYSTEM_NPM	2✔
310	case purl.TypeGolang:	2✔
311	return pbinternal.DepEcosystem_DEP_ECOSYSTEM_GO	2✔
UNCOV 312	default:	×
313	return pbinternal.DepEcosystem_DEP_ECOSYSTEM_UNSPECIFIED	×
314	}
315	}
316
317	// ingestFileForFullDiff processes a given file's patch from a pull request.
318	// It scans through the patch line by line, identifying the changes made.
319	// If it's a hunk header, it extracts the starting line number. If it's an addition, it records the line content and its number.
320	// The function also increments the line number for context lines (lines that provide context but haven't been modified).
UNCOV 321	func ingestFileForFullDiff(filename, patch, patchUrl string) (*pbinternal.PrContents_File, error) {	×
322	var result []*pbinternal.PrContents_File_Line	×
323		×
324	scanner := bufio.NewScanner(strings.NewReader(patch))	×
325	regex := regexp.MustCompile(`@@ -\d+,\d+ \+(\d+),\d+ @@`)	×
326		×
327	var currentLineNumber int64	×
328	var err error	×
329	for scanner.Scan() {	×
330	line := scanner.Text()	×
331		×
332	if matches := regex.FindStringSubmatch(line); matches != nil {	×
333	currentLineNumber, err = strconv.ParseInt(matches[1], 10, 32)	×
334	if err != nil {	×
335	return nil, fmt.Errorf("error parsing line number from the hunk header: %w", err)	×
336	}	×
337	} else if strings.HasPrefix(line, "+") {	×
338	result = append(result, &pbinternal.PrContents_File_Line{	×
339	Content: line[1:],	×
340	// see the use of strconv.ParseInt above: this is a safe downcast	×
341	// nolint: gosec	×
342	LineNumber: int32(currentLineNumber),	×
343	})	×
344		×
345	currentLineNumber++	×
346	} else if !strings.HasPrefix(line, "-") {	×
347	currentLineNumber++	×
348	}	×
349	}
350
UNCOV 351	if err := scanner.Err(); err != nil {	×
352	return nil, fmt.Errorf("error reading patch: %w", err)	×
353	}	×
354
UNCOV 355	return &pbinternal.PrContents_File{	×
356	Name: filename,	×
357	FilePatchUrl: patchUrl,	×
358	PatchLines: result,	×
359	}, nil	×
360	}
361
362	func (di *Diff) getEcosystemForFile(filename string) DependencyEcosystem {	5✔
363	lastComponent := filepath.Base(filename)	5✔
364		5✔
365	for _, ecoMapping := range di.cfg.Ecosystems {	10✔
366	if match, _ := filepath.Match(ecoMapping.Depfile, lastComponent); match {	8✔
367	return DependencyEcosystem(ecoMapping.Name)	3✔
368	}	3✔
369	}
370	return DepEcosystemNone	2✔
371	}
372
UNCOV 373	func (di *Diff) getParserForFile(filename string, logger zerolog.Logger) ecosystemParser {	×
374	eco := di.getEcosystemForFile(filename)	×
375	if eco == DepEcosystemNone {	×
376	logger.Debug().	×
377	Str("filename", filename).	×
378	Msg("No ecosystem found, skipping")	×
379	return nil	×
380	}	×
381
UNCOV 382	logger.Debug().	×
383	Str("filename", filename).	×
384	Str("package-ecosystem", string(eco)).	×
385	Msg("matched ecosystem")	×
386		×
387	return newEcosystemParser(eco)	×
388	}
389
390	// Computes the set of elements in updated which are not in base.
391	// Note: this function may permute (sort) the order of elements in base and updated.
392	func setDifference[Slice ~[]E, E any](base Slice, updated Slice, sorter func(a, b E) int) Slice {	8✔
393		8✔
394	slices.SortFunc(base, sorter)	8✔
395	slices.SortFunc(updated, sorter)	8✔
396		8✔
397	baseIdx, newIdx := 0, 0	8✔
398	ret := make(Slice, 0)	8✔
399	for baseIdx < len(base) && newIdx < len(updated) {	25✔
400	cmpResult := sorter(base[baseIdx], updated[newIdx])	17✔
401	if cmpResult < 0 {	23✔
402	baseIdx++	6✔
403	} else if cmpResult > 0 {	22✔
404	ret = append(ret, updated[newIdx])	5✔
405	newIdx++	5✔
406	} else {	11✔
407	baseIdx++	6✔
408	newIdx++	6✔
409	}	6✔
410	}
411	if newIdx < len(updated) {	11✔
412	ret = append(ret, updated[newIdx:]...)	3✔
413	}	3✔
414
415	// TODO: add metric for number of deps scanned vs total deps
416
417	return ret	8✔
418	}

mindersec / minder / 15981216547

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous