29869011232

Committed 21 Jul 2026 09:12PM UTC coverage: 60.908%. First build

Build # 29869011232

Build Type

Pull #6179

github

Committed by

web-flow

Commit Message

Merge 3814c5f59 into 387255c89

Pull Request Pull Request #6179: build(deps): bump github.com/google/osv-scalibr from 0.3.4 to 0.4.5

Coverage Stats

91 of 130 new or added lines in 2 files covered. (70.0%)

21095 of 34634 relevant lines covered (60.91%)

40.12 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.48

/internal/engine/ingester/diff/diff.go

// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
// SPDX-License-Identifier: Apache-2.0

// Package diff provides the diff rule data ingest engine
package diff

import (
        "bufio"
        "cmp"
        "context"
        "fmt"
        "math"
        "os"
        "path/filepath"
        "regexp"
        "slices"
        "strconv"
        "strings"

        "github.com/go-git/go-billy/v5"
        "github.com/go-git/go-billy/v5/helper/iofs"
        scalibr "github.com/google/osv-scalibr"
        scalibr_cfg "github.com/google/osv-scalibr/binary/proto/config_go_proto"
        "github.com/google/osv-scalibr/extractor"
        scalibr_fs "github.com/google/osv-scalibr/fs"
        scalibr_plugin "github.com/google/osv-scalibr/plugin"
        "github.com/google/osv-scalibr/plugin/list"
        "github.com/google/osv-scalibr/purl"
        "github.com/rs/zerolog"
        "google.golang.org/protobuf/reflect/protoreflect"

        pbinternal "github.com/mindersec/minder/internal/proto"
        pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
        "github.com/mindersec/minder/pkg/engine/v1/interfaces"
        "github.com/mindersec/minder/pkg/entities/v1/checkpoints"
)

const (
        // DiffRuleDataIngestType is the type of the diff rule data ingest engine
        DiffRuleDataIngestType = "diff"
        prFilesPerPage         = 30
        wildcard               = "*"
)

// Diff is the diff rule data ingest engine
type Diff struct {
        cli interfaces.GitHubListAndClone
        cfg *pb.DiffType
}

// NewDiffIngester creates a new diff ingester
func NewDiffIngester(
        cfg *pb.DiffType,
        cli interfaces.GitHubListAndClone,
) (*Diff, error) {
        if cfg == nil {
                cfg = &pb.DiffType{}
        }

        if cli == nil {
                return nil, fmt.Errorf("provider is nil")
        }

        return &Diff{
                cfg: cfg,
                cli: cli,
        }, nil
}

// GetType returns the type of the diff rule data ingest engine
func (*Diff) GetType() string {
        return DiffRuleDataIngestType
}

// GetConfig returns the config for the diff rule data ingest engine
func (di *Diff) GetConfig() protoreflect.ProtoMessage {
        return di.cfg
}

// Ingest ingests a diff from a pull request in accordance with its type
func (di *Diff) Ingest(
        ctx context.Context,
        ent protoreflect.ProtoMessage,
        _ map[string]any,
) (*interfaces.Ingested, error) {
        pr, ok := ent.(*pbinternal.PullRequest)
        if !ok {
                return nil, fmt.Errorf("entity is not a pull request")
        }

        // The GitHub Go API takes an int32, but our proto stores an int64; make sure we don't overflow
        if pr.Number > math.MaxInt {
                return nil, fmt.Errorf("pr number is too large")
        }
        prNumber := int(pr.Number)

        switch di.cfg.GetType() {
        case "", pb.DiffTypeDep:
                return di.getDepTypeDiff(ctx, prNumber, pr)

        case pb.DiffTypeNewDeps:
                // TODO: once we've tested some, convert DiffTypeDep to use this algorithm.
                return di.getScalibrTypeDiff(ctx, prNumber, pr)

        case pb.DiffTypeFull:
                return di.getFullTypeDiff(ctx, prNumber, pr)

        default:
                return nil, fmt.Errorf("unknown diff type")
        }
}

func (di *Diff) getDepTypeDiff(ctx context.Context, prNumber int, pr *pbinternal.PullRequest) (*interfaces.Ingested, error) {
        deps := pbinternal.PrDependencies{Pr: pr}
        page := 0

        for {
                prFiles, resp, err := di.cli.ListFiles(ctx, pr.RepoOwner, pr.RepoName, prNumber, prFilesPerPage, page)
                if err != nil {
                        return nil, fmt.Errorf("error getting pull request files: %w", err)
                }

                for _, file := range prFiles {
                        fileDiffs, err := di.ingestFileForDepDiff(file.GetFilename(), file.GetPatch(), file.GetRawURL(), *zerolog.Ctx(ctx))
                        if err != nil {
                                return nil, fmt.Errorf("error ingesting file %s: %w", file.GetFilename(), err)
                        }
                        deps.Deps = append(deps.Deps, fileDiffs...)
                }

                if resp.NextPage == 0 {
                        break
                }

                page = resp.NextPage
        }

        return &interfaces.Ingested{Object: &deps, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil
}

func (di *Diff) getFullTypeDiff(ctx context.Context, prNumber int, pr *pbinternal.PullRequest) (*interfaces.Ingested, error) {
        diff := &pbinternal.PrContents{Pr: pr}
        page := 0

        for {
                prFiles, resp, err := di.cli.ListFiles(ctx, pr.RepoOwner, pr.RepoName, prNumber, prFilesPerPage, page)
                if err != nil {
                        return nil, fmt.Errorf("error getting pull request files: %w", err)
                }

                for _, file := range prFiles {
                        fileDiffs, err := ingestFileForFullDiff(file.GetFilename(), file.GetPatch(), file.GetRawURL())
                        if err != nil {
                                return nil, fmt.Errorf("error ingesting file %s: %w", file.GetFilename(), err)
                        }
                        diff.Files = append(diff.Files, fileDiffs)
                }

                if resp.NextPage == 0 {
                        break
                }

                page = resp.NextPage
        }

        return &interfaces.Ingested{Object: diff, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil
}

func (di *Diff) ingestFileForDepDiff(
        filename, patchContents, patchUrl string,
        logger zerolog.Logger,
) ([]*pbinternal.PrDependencies_ContextualDependency, error) {
        parser := di.getParserForFile(filename, logger)
        if parser == nil {
                return nil, nil
        }

        depBatch, err := parser(patchContents)
        if err != nil {
                return nil, fmt.Errorf("error parsing file %s: %w", filename, err)
        }

        batchCtxDeps := make([]*pbinternal.PrDependencies_ContextualDependency, 0, len(depBatch))
        for i := range depBatch {
                dep := depBatch[i]
                batchCtxDeps = append(batchCtxDeps, &pbinternal.PrDependencies_ContextualDependency{
                        Dep: dep,
                        File: &pbinternal.PrDependencies_ContextualDependency_FilePatch{
                                Name:     filename,
                                PatchUrl: patchUrl,
                        },
                })
        }

        return batchCtxDeps, nil
}

func (di *Diff) getScalibrTypeDiff(ctx context.Context, _ int, pr *pbinternal.PullRequest) (*interfaces.Ingested, error) {
        deps := pbinternal.PrDependencies{Pr: pr}

        // TODO: we should be able to just fetch the additional commits between base and target.
        // Our current Git abstraction isn't quite powerful enough, so we do two shallow clones.

        baseInventory, err := di.scalibrInventory(ctx, pr.BaseCloneUrl, pr.BaseRef)
        if err != nil {
                return nil, fmt.Errorf("failed to clone base from %s at %q: %w", pr.BaseCloneUrl, pr.BaseRef, err)
        }
        newInventory, err := di.scalibrInventory(ctx, pr.TargetCloneUrl, pr.TargetRef)
        if err != nil {
                return nil, fmt.Errorf("failed to clone fork from %s at %q: %w", pr.TargetCloneUrl, pr.TargetRef, err)
        }

        newDeps := setDifference(baseInventory, newInventory, inventorySorter)

        deps.Deps = make([]*pbinternal.PrDependencies_ContextualDependency, 0, len(newDeps))
        for _, inventory := range newDeps {
                for _, filename := range inventory.Locations {
                        deps.Deps = append(deps.Deps, &pbinternal.PrDependencies_ContextualDependency{
                                Dep: &pbinternal.Dependency{
                                        Ecosystem: inventoryToEcosystem(inventory),
                                        Name:      inventory.Name,
                                        Version:   inventory.Version,
                                },
                                File: &pbinternal.PrDependencies_ContextualDependency_FilePatch{
                                        Name:     filename,
                                        PatchUrl: "", // TODO: do we need this?
                                },
                        })
                }
        }

        return &interfaces.Ingested{Object: &deps, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil
}

func inventorySorter(a *extractor.Package, b *extractor.Package) int {
        // If we compare by name and version first, we can avoid serializing Locations to strings
        res := cmp.Or(cmp.Compare(a.Name, b.Name), cmp.Compare(a.Version, b.Version))
        if res != 0 {
                return res
        }
        // TODO: Locations should probably be sorted, but scalibr is going to export a compare function.
        aLoc := fmt.Sprintf("%v", a.Locations)
        bLoc := fmt.Sprintf("%v", b.Locations)
        return cmp.Compare(aLoc, bLoc)
}

func (di *Diff) scalibrInventory(ctx context.Context, repoURL string, ref string) ([]*extractor.Package, error) {
        clone, err := di.cli.Clone(ctx, repoURL, ref)
        if err != nil {
                return nil, err
        }

        tree, err := clone.Worktree()
        if err != nil {
                return nil, err
        }
        return scanFs(ctx, tree.Filesystem, map[string]string{})
}

func scanFs(ctx context.Context, memFS billy.Filesystem, _ map[string]string) ([]*extractor.Package, error) {
        // have to down-cast here, because scalibr needs multiple io/fs types
        wrapped, ok := iofs.New(memFS).(scalibr_fs.FS)
        if !ok {
                return nil, fmt.Errorf("error converting filesystem to ReadDirFS")
        }

        desiredCaps := scalibr_plugin.Capabilities{
                OS:            scalibr_plugin.OSLinux,
                Network:       scalibr_plugin.NetworkOffline,
                DirectFS:      false,
                RunningSystem: false,
        }

        // TODO: it's unfortunate that scalibr spills files to disk.  File an upstream bug?
        // NOTE: since we require NetworkOffline, we may not actually download anything...
        tmpDir, err := os.MkdirTemp("", "minder-scalibr-*")
        if err != nil {
                return nil, fmt.Errorf("failed to create temporary scalibr directory: %w", err)
        }
        defer func() {
                _ = os.RemoveAll(tmpDir)
        }()
        cfg := scalibr_cfg.PluginConfig{
                MaxFileSizeBytes:  1024 * 1024,
                LocalRegistry:     tmpDir,
                DisableGoogleAuth: true,
        }

        scalibrFs := scalibr_fs.ScanRoot{FS: wrapped}
        plugins, err := list.FromCapabilities(&desiredCaps, &cfg)
        if err != nil {
                return nil, err
        }
        // unknownbinariesextr uses file extension to determine "binary-ness", and triggers on e.g. .py files
        skipPlugins := []string{"ffa/unknownbinariesextr"}
        plugins = slices.DeleteFunc(plugins, func(p scalibr_plugin.Plugin) bool {
                return slices.Contains(skipPlugins, p.Name())
        })
        scanConfig := scalibr.ScanConfig{
                ScanRoots:    []*scalibr_fs.ScanRoot{&scalibrFs},
                Plugins:      plugins,
                Capabilities: &desiredCaps,
        }

        scanner := scalibr.New()
        scanResults := scanner.Scan(ctx, &scanConfig)

        if scanResults == nil || scanResults.Status == nil {
                return nil, fmt.Errorf("error scanning files: no results")
        }
        switch scanResults.Status.Status {
        case scalibr_plugin.ScanStatusSucceeded:
                return scanResults.Inventory.Packages, nil
                // Scalibr runs a lot of plugins and aggregates the result.  Some of these are picky, and
                // fail for random reasons.  Accept partial success, but log the failing plugins.
        case scalibr_plugin.ScanStatusPartiallySucceeded:
                known_bad := []string{
                        "endoflife/linuxdistro", // https://github.com/google/osv-scalibr/pull/2068
                        "rust/cargoauditable",   // https://github.com/go-git/go-billy/pull/208
                }
                for _, ps := range scanResults.PluginStatus {
                        if ps.Status.Status != scalibr_plugin.ScanStatusSucceeded {
                                if !slices.Contains(known_bad, ps.Name) {
                                        zerolog.Ctx(ctx).Warn().Str("plugin", ps.Name).Str("status", ps.Status.FailureReason).
                                                Msg("Scalibr plugin failed")
                                }
                        }
                }
                return scanResults.Inventory.Packages, nil
        case scalibr_plugin.ScanStatusUnspecified, scalibr_plugin.ScanStatusFailed:
                fallthrough
        default:
                return nil, fmt.Errorf("error scanning files: %s", scanResults.Status)
        }
}

func inventoryToEcosystem(inventory *extractor.Package) pbinternal.DepEcosystem {
        if inventory == nil {
                zerolog.Ctx(context.Background()).Warn().Msg("nil ecosystem scanning diffs")
                return pbinternal.DepEcosystem_DEP_ECOSYSTEM_UNSPECIFIED
        }

        package_url := inventory.PURL()
        if package_url == nil {
                package_url = &purl.PackageURL{}
        }

        // Sometimes Scalibr uses the string "PyPI" instead of "pypi" when reporting the ecosystem.
        switch package_url.Type {
        // N.B. using an enum here abitrarily restricts our ability to add new
        // ecosystems without a core minder change.  Switching to strings ala
        // purl might be an improvement.
        case purl.TypePyPi:
                return pbinternal.DepEcosystem_DEP_ECOSYSTEM_PYPI
        case purl.TypeNPM:
                return pbinternal.DepEcosystem_DEP_ECOSYSTEM_NPM
        case purl.TypeGolang:
                return pbinternal.DepEcosystem_DEP_ECOSYSTEM_GO
        default:
                return pbinternal.DepEcosystem_DEP_ECOSYSTEM_UNSPECIFIED
        }
}

// ingestFileForFullDiff processes a given file's patch from a pull request.
// It scans through the patch line by line, identifying the changes made.
// If it's a hunk header, it extracts the starting line number. If it's an addition, it records the line content and its number.
// The function also increments the line number for context lines (lines that provide context but haven't been modified).
func ingestFileForFullDiff(filename, patch, patchUrl string) (*pbinternal.PrContents_File, error) {
        var result []*pbinternal.PrContents_File_Line

        scanner := bufio.NewScanner(strings.NewReader(patch))
        regex := regexp.MustCompile(`@@ -\d+,\d+ \+(\d+),\d+ @@`)

        var currentLineNumber int64
        var err error
        for scanner.Scan() {
                line := scanner.Text()

                if matches := regex.FindStringSubmatch(line); matches != nil {
                        currentLineNumber, err = strconv.ParseInt(matches[1], 10, 32)
                        if err != nil {
                                return nil, fmt.Errorf("error parsing line number from the hunk header: %w", err)
                        }
                } else if strings.HasPrefix(line, "+") {
                        result = append(result, &pbinternal.PrContents_File_Line{
                                Content: line[1:],
                                // see the use of strconv.ParseInt above: this is a safe downcast
                                // nolint: gosec
                                LineNumber: int32(currentLineNumber),
                        })

                        currentLineNumber++
                } else if !strings.HasPrefix(line, "-") {
                        currentLineNumber++
                }
        }

        if err := scanner.Err(); err != nil {
                return nil, fmt.Errorf("error reading patch: %w", err)
        }

        return &pbinternal.PrContents_File{
                Name:         filename,
                FilePatchUrl: patchUrl,
                PatchLines:   result,
        }, nil
}

func (di *Diff) getEcosystemForFile(filename string) DependencyEcosystem {
        lastComponent := filepath.Base(filename)

        for _, ecoMapping := range di.cfg.Ecosystems {
                if match, _ := filepath.Match(ecoMapping.Depfile, lastComponent); match {
                        return DependencyEcosystem(ecoMapping.Name)
                }
        }
        return DepEcosystemNone
}

func (di *Diff) getParserForFile(filename string, logger zerolog.Logger) ecosystemParser {
        eco := di.getEcosystemForFile(filename)
        if eco == DepEcosystemNone {
                logger.Debug().
                        Str("filename", filename).
                        Msg("No ecosystem found, skipping")
                return nil
        }

        logger.Debug().
                Str("filename", filename).
                Str("package-ecosystem", string(eco)).
                Msg("matched ecosystem")

        return newEcosystemParser(eco)
}

// Computes the set of elements in updated which are not in base.
// Note: this function may permute (sort) the order of elements in base and updated.
func setDifference[Slice ~[]E, E any](base Slice, updated Slice, sorter func(a, b E) int) Slice {

        slices.SortFunc(base, sorter)
        slices.SortFunc(updated, sorter)

        baseIdx, newIdx := 0, 0
        ret := make(Slice, 0)
        for baseIdx < len(base) && newIdx < len(updated) {
                cmpResult := sorter(base[baseIdx], updated[newIdx])
                if cmpResult < 0 {
                        baseIdx++
                } else if cmpResult > 0 {
                        ret = append(ret, updated[newIdx])
                        newIdx++
                } else {
                        baseIdx++
                        newIdx++
                }
        }
        if newIdx < len(updated) {
                ret = append(ret, updated[newIdx:]...)
        }

        // TODO: add metric for number of deps scanned vs total deps

        return ret
}

1	// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
2	// SPDX-License-Identifier: Apache-2.0
3
4	// Package diff provides the diff rule data ingest engine
5	package diff
6
7	import (
8	"bufio"
9	"cmp"
10	"context"
11	"fmt"
12	"math"
13	"os"
14	"path/filepath"
15	"regexp"
16	"slices"
17	"strconv"
18	"strings"
19
20	"github.com/go-git/go-billy/v5"
21	"github.com/go-git/go-billy/v5/helper/iofs"
22	scalibr "github.com/google/osv-scalibr"
23	scalibr_cfg "github.com/google/osv-scalibr/binary/proto/config_go_proto"
24	"github.com/google/osv-scalibr/extractor"
25	scalibr_fs "github.com/google/osv-scalibr/fs"
26	scalibr_plugin "github.com/google/osv-scalibr/plugin"
27	"github.com/google/osv-scalibr/plugin/list"
28	"github.com/google/osv-scalibr/purl"
29	"github.com/rs/zerolog"
30	"google.golang.org/protobuf/reflect/protoreflect"
31
32	pbinternal "github.com/mindersec/minder/internal/proto"
33	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
34	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
35	"github.com/mindersec/minder/pkg/entities/v1/checkpoints"
36	)
37
38	const (
39	// DiffRuleDataIngestType is the type of the diff rule data ingest engine
40	DiffRuleDataIngestType = "diff"
41	prFilesPerPage = 30
42	wildcard = "*"
43	)
44
45	// Diff is the diff rule data ingest engine
46	type Diff struct {
47	cli interfaces.GitHubListAndClone
48	cfg *pb.DiffType
49	}
50
51	// NewDiffIngester creates a new diff ingester
52	func NewDiffIngester(
53	cfg *pb.DiffType,
54	cli interfaces.GitHubListAndClone,
55	) (*Diff, error) {	4✔
56	if cfg == nil {	4✔
57	cfg = &pb.DiffType{}	×
58	}	×
59
60	if cli == nil {	4✔
61	return nil, fmt.Errorf("provider is nil")	×
62	}	×
63
64	return &Diff{	4✔
65	cfg: cfg,	4✔
66	cli: cli,	4✔
67	}, nil	4✔
68	}
69
70	// GetType returns the type of the diff rule data ingest engine
71	func (*Diff) GetType() string {	3✔
72	return DiffRuleDataIngestType	3✔
73	}	3✔
74
75	// GetConfig returns the config for the diff rule data ingest engine
76	func (di *Diff) GetConfig() protoreflect.ProtoMessage {	6✔
77	return di.cfg	6✔
78	}	6✔
79
80	// Ingest ingests a diff from a pull request in accordance with its type
81	func (di *Diff) Ingest(
82	ctx context.Context,
83	ent protoreflect.ProtoMessage,
84	_ map[string]any,
85	) (*interfaces.Ingested, error) {	4✔
86	pr, ok := ent.(*pbinternal.PullRequest)	4✔
87	if !ok {	4✔
88	return nil, fmt.Errorf("entity is not a pull request")	×
89	}	×
90
91	// The GitHub Go API takes an int32, but our proto stores an int64; make sure we don't overflow
92	if pr.Number > math.MaxInt {	4✔
93	return nil, fmt.Errorf("pr number is too large")	×
94	}	×
95	prNumber := int(pr.Number)	4✔
96		4✔
97	switch di.cfg.GetType() {	4✔
98	case "", pb.DiffTypeDep:	×
99	return di.getDepTypeDiff(ctx, prNumber, pr)	×
100
101	case pb.DiffTypeNewDeps:	4✔
102	// TODO: once we've tested some, convert DiffTypeDep to use this algorithm.	4✔
103	return di.getScalibrTypeDiff(ctx, prNumber, pr)	4✔
104
105	case pb.DiffTypeFull:	×
106	return di.getFullTypeDiff(ctx, prNumber, pr)	×
107
108	default:	×
109	return nil, fmt.Errorf("unknown diff type")	×
110	}
111	}
112
113	func (di Diff) getDepTypeDiff(ctx context.Context, prNumber int, pr pbinternal.PullRequest) (*interfaces.Ingested, error) {	×
114	deps := pbinternal.PrDependencies{Pr: pr}	×
115	page := 0	×
116		×
117	for {	×
118	prFiles, resp, err := di.cli.ListFiles(ctx, pr.RepoOwner, pr.RepoName, prNumber, prFilesPerPage, page)	×
119	if err != nil {	×
120	return nil, fmt.Errorf("error getting pull request files: %w", err)	×
121	}	×
122
123	for _, file := range prFiles {	×
124	fileDiffs, err := di.ingestFileForDepDiff(file.GetFilename(), file.GetPatch(), file.GetRawURL(), *zerolog.Ctx(ctx))	×
125	if err != nil {	×
126	return nil, fmt.Errorf("error ingesting file %s: %w", file.GetFilename(), err)	×
127	}	×
128	deps.Deps = append(deps.Deps, fileDiffs...)	×
129	}
130
131	if resp.NextPage == 0 {	×
132	break	×
133	}
134
135	page = resp.NextPage	×
136	}
137
138	return &interfaces.Ingested{Object: &deps, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil	×
139	}
140
141	func (di Diff) getFullTypeDiff(ctx context.Context, prNumber int, pr pbinternal.PullRequest) (*interfaces.Ingested, error) {	×
142	diff := &pbinternal.PrContents{Pr: pr}	×
143	page := 0	×
144		×
145	for {	×
146	prFiles, resp, err := di.cli.ListFiles(ctx, pr.RepoOwner, pr.RepoName, prNumber, prFilesPerPage, page)	×
147	if err != nil {	×
148	return nil, fmt.Errorf("error getting pull request files: %w", err)	×
149	}	×
150
151	for _, file := range prFiles {	×
152	fileDiffs, err := ingestFileForFullDiff(file.GetFilename(), file.GetPatch(), file.GetRawURL())	×
153	if err != nil {	×
154	return nil, fmt.Errorf("error ingesting file %s: %w", file.GetFilename(), err)	×
155	}	×
156	diff.Files = append(diff.Files, fileDiffs)	×
157	}
158
159	if resp.NextPage == 0 {	×
160	break	×
161	}
162
163	page = resp.NextPage	×
164	}
165
166	return &interfaces.Ingested{Object: diff, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil	×
167	}
168
169	func (di *Diff) ingestFileForDepDiff(
170	filename, patchContents, patchUrl string,
171	logger zerolog.Logger,
172	) ([]*pbinternal.PrDependencies_ContextualDependency, error) {	×
173	parser := di.getParserForFile(filename, logger)	×
174	if parser == nil {	×
175	return nil, nil	×
176	}	×
177
178	depBatch, err := parser(patchContents)	×
179	if err != nil {	×
180	return nil, fmt.Errorf("error parsing file %s: %w", filename, err)	×
181	}	×
182
183	batchCtxDeps := make([]*pbinternal.PrDependencies_ContextualDependency, 0, len(depBatch))	×
184	for i := range depBatch {	×
185	dep := depBatch[i]	×
186	batchCtxDeps = append(batchCtxDeps, &pbinternal.PrDependencies_ContextualDependency{	×
187	Dep: dep,	×
188	File: &pbinternal.PrDependencies_ContextualDependency_FilePatch{	×
189	Name: filename,	×
190	PatchUrl: patchUrl,	×
191	},	×
192	})	×
193	}	×
194
195	return batchCtxDeps, nil	×
196	}
197
198	func (di Diff) getScalibrTypeDiff(ctx context.Context, _ int, pr pbinternal.PullRequest) (*interfaces.Ingested, error) {	4✔
199	deps := pbinternal.PrDependencies{Pr: pr}	4✔
200		4✔
201	// TODO: we should be able to just fetch the additional commits between base and target.	4✔
202	// Our current Git abstraction isn't quite powerful enough, so we do two shallow clones.	4✔
203		4✔
204	baseInventory, err := di.scalibrInventory(ctx, pr.BaseCloneUrl, pr.BaseRef)	4✔
205	if err != nil {	4✔
206	return nil, fmt.Errorf("failed to clone base from %s at %q: %w", pr.BaseCloneUrl, pr.BaseRef, err)	×
207	}	×
208	newInventory, err := di.scalibrInventory(ctx, pr.TargetCloneUrl, pr.TargetRef)	4✔
209	if err != nil {	4✔
210	return nil, fmt.Errorf("failed to clone fork from %s at %q: %w", pr.TargetCloneUrl, pr.TargetRef, err)	×
211	}	×
212
213	newDeps := setDifference(baseInventory, newInventory, inventorySorter)	4✔
214		4✔
215	deps.Deps = make([]*pbinternal.PrDependencies_ContextualDependency, 0, len(newDeps))	4✔
216	for _, inventory := range newDeps {	10✔
217	for _, filename := range inventory.Locations {	12✔
218	deps.Deps = append(deps.Deps, &pbinternal.PrDependencies_ContextualDependency{	6✔
219	Dep: &pbinternal.Dependency{	6✔
220	Ecosystem: inventoryToEcosystem(inventory),	6✔
221	Name: inventory.Name,	6✔
222	Version: inventory.Version,	6✔
223	},	6✔
224	File: &pbinternal.PrDependencies_ContextualDependency_FilePatch{	6✔
225	Name: filename,	6✔
226	PatchUrl: "", // TODO: do we need this?	6✔
227	},	6✔
228	})	6✔
229	}	6✔
230	}
231
232	return &interfaces.Ingested{Object: &deps, Checkpoint: checkpoints.NewCheckpointV1Now()}, nil	4✔
233	}
234
235	func inventorySorter(a extractor.Package, b extractor.Package) int {	22✔
236	// If we compare by name and version first, we can avoid serializing Locations to strings	22✔
237	res := cmp.Or(cmp.Compare(a.Name, b.Name), cmp.Compare(a.Version, b.Version))	22✔
238	if res != 0 {	42✔
239	return res	20✔
240	}	20✔
241	// TODO: Locations should probably be sorted, but scalibr is going to export a compare function.
242	aLoc := fmt.Sprintf("%v", a.Locations)	2✔
243	bLoc := fmt.Sprintf("%v", b.Locations)	2✔
244	return cmp.Compare(aLoc, bLoc)	2✔
245	}
246
247	func (di Diff) scalibrInventory(ctx context.Context, repoURL string, ref string) ([]extractor.Package, error) {	8✔
248	clone, err := di.cli.Clone(ctx, repoURL, ref)	8✔
249	if err != nil {	8✔
250	return nil, err	×
251	}	×
252
253	tree, err := clone.Worktree()	8✔
254	if err != nil {	8✔
255	return nil, err	×
256	}	×
257	return scanFs(ctx, tree.Filesystem, map[string]string{})	8✔
258	}
259
260	func scanFs(ctx context.Context, memFS billy.Filesystem, _ map[string]string) ([]*extractor.Package, error) {	8✔
261	// have to down-cast here, because scalibr needs multiple io/fs types	8✔
262	wrapped, ok := iofs.New(memFS).(scalibr_fs.FS)	8✔
263	if !ok {	8✔
264	return nil, fmt.Errorf("error converting filesystem to ReadDirFS")	×
265	}	×
266
267	desiredCaps := scalibr_plugin.Capabilities{	8✔
268	OS: scalibr_plugin.OSLinux,	8✔
269	Network: scalibr_plugin.NetworkOffline,	8✔
270	DirectFS: false,	8✔
271	RunningSystem: false,	8✔
272	}	8✔
273		8✔
274	// TODO: it's unfortunate that scalibr spills files to disk. File an upstream bug?	8✔
275	// NOTE: since we require NetworkOffline, we may not actually download anything...	8✔
276	tmpDir, err := os.MkdirTemp("", "minder-scalibr-*")	8✔
277	if err != nil {	8✔
NEW 278	return nil, fmt.Errorf("failed to create temporary scalibr directory: %w", err)	×
NEW 279	}	×
280	defer func() {	16✔
281	_ = os.RemoveAll(tmpDir)	8✔
282	}()	8✔
283	cfg := scalibr_cfg.PluginConfig{	8✔
284	MaxFileSizeBytes: 1024 * 1024,	8✔
285	LocalRegistry: tmpDir,	8✔
286	DisableGoogleAuth: true,	8✔
287	}	8✔
288		8✔
289	scalibrFs := scalibr_fs.ScanRoot{FS: wrapped}	8✔
290	plugins, err := list.FromCapabilities(&desiredCaps, &cfg)	8✔
291	if err != nil {	8✔
NEW 292	return nil, err	×
NEW 293	}	×
294	// unknownbinariesextr uses file extension to determine "binary-ness", and triggers on e.g. .py files
295	skipPlugins := []string{"ffa/unknownbinariesextr"}	8✔
296	plugins = slices.DeleteFunc(plugins, func(p scalibr_plugin.Plugin) bool {	2,032✔
297	return slices.Contains(skipPlugins, p.Name())	2,024✔
298	})	2,024✔
299	scanConfig := scalibr.ScanConfig{	8✔
300	ScanRoots: []*scalibr_fs.ScanRoot{&scalibrFs},	8✔
301	Plugins: plugins,	8✔
302	Capabilities: &desiredCaps,	8✔
303	}	8✔
304		8✔
305	scanner := scalibr.New()	8✔
306	scanResults := scanner.Scan(ctx, &scanConfig)	8✔
307		8✔
308	if scanResults == nil \|\| scanResults.Status == nil {	8✔
309	return nil, fmt.Errorf("error scanning files: no results")	×
310	}	×
311	switch scanResults.Status.Status {	8✔
NEW 312	case scalibr_plugin.ScanStatusSucceeded:	×
NEW 313	return scanResults.Inventory.Packages, nil	×
314	// Scalibr runs a lot of plugins and aggregates the result. Some of these are picky, and
315	// fail for random reasons. Accept partial success, but log the failing plugins.
316	case scalibr_plugin.ScanStatusPartiallySucceeded:	8✔
317	known_bad := []string{	8✔
318	"endoflife/linuxdistro", // https://github.com/google/osv-scalibr/pull/2068	8✔
319	"rust/cargoauditable", // https://github.com/go-git/go-billy/pull/208	8✔
320	}	8✔
321	for _, ps := range scanResults.PluginStatus {	896✔
322	if ps.Status.Status != scalibr_plugin.ScanStatusSucceeded {	897✔
323	if !slices.Contains(known_bad, ps.Name) {	9✔
NEW 324	zerolog.Ctx(ctx).Warn().Str("plugin", ps.Name).Str("status", ps.Status.FailureReason).	×
NEW 325	Msg("Scalibr plugin failed")	×
NEW 326	}	×
327	}
328	}
329	return scanResults.Inventory.Packages, nil	8✔
NEW 330	case scalibr_plugin.ScanStatusUnspecified, scalibr_plugin.ScanStatusFailed:	×
NEW 331	fallthrough	×
NEW 332	default:	×
333	return nil, fmt.Errorf("error scanning files: %s", scanResults.Status)	×
334	}
335	}
336
337	func inventoryToEcosystem(inventory *extractor.Package) pbinternal.DepEcosystem {	6✔
338	if inventory == nil {	6✔
339	zerolog.Ctx(context.Background()).Warn().Msg("nil ecosystem scanning diffs")	×
340	return pbinternal.DepEcosystem_DEP_ECOSYSTEM_UNSPECIFIED	×
341	}	×
342
343	package_url := inventory.PURL()	6✔
344	if package_url == nil {	6✔
NEW 345	package_url = &purl.PackageURL{}	×
NEW 346	}	×
347
348	// Sometimes Scalibr uses the string "PyPI" instead of "pypi" when reporting the ecosystem.
349	switch package_url.Type {	6✔
350	// N.B. using an enum here abitrarily restricts our ability to add new
351	// ecosystems without a core minder change. Switching to strings ala
352	// purl might be an improvement.
353	case purl.TypePyPi:	2✔
354	return pbinternal.DepEcosystem_DEP_ECOSYSTEM_PYPI	2✔
355	case purl.TypeNPM:	2✔
356	return pbinternal.DepEcosystem_DEP_ECOSYSTEM_NPM	2✔
357	case purl.TypeGolang:	2✔
358	return pbinternal.DepEcosystem_DEP_ECOSYSTEM_GO	2✔
359	default:	×
360	return pbinternal.DepEcosystem_DEP_ECOSYSTEM_UNSPECIFIED	×
361	}
362	}
363
364	// ingestFileForFullDiff processes a given file's patch from a pull request.
365	// It scans through the patch line by line, identifying the changes made.
366	// If it's a hunk header, it extracts the starting line number. If it's an addition, it records the line content and its number.
367	// The function also increments the line number for context lines (lines that provide context but haven't been modified).
368	func ingestFileForFullDiff(filename, patch, patchUrl string) (*pbinternal.PrContents_File, error) {	×
369	var result []*pbinternal.PrContents_File_Line	×
370		×
371	scanner := bufio.NewScanner(strings.NewReader(patch))	×
372	regex := regexp.MustCompile(`@@ -\d+,\d+ \+(\d+),\d+ @@`)	×
373		×
374	var currentLineNumber int64	×
375	var err error	×
376	for scanner.Scan() {	×
377	line := scanner.Text()	×
378		×
379	if matches := regex.FindStringSubmatch(line); matches != nil {	×
380	currentLineNumber, err = strconv.ParseInt(matches[1], 10, 32)	×
381	if err != nil {	×
382	return nil, fmt.Errorf("error parsing line number from the hunk header: %w", err)	×
383	}	×
384	} else if strings.HasPrefix(line, "+") {	×
385	result = append(result, &pbinternal.PrContents_File_Line{	×
386	Content: line[1:],	×
387	// see the use of strconv.ParseInt above: this is a safe downcast	×
388	// nolint: gosec	×
389	LineNumber: int32(currentLineNumber),	×
390	})	×
391		×
392	currentLineNumber++	×
393	} else if !strings.HasPrefix(line, "-") {	×
394	currentLineNumber++	×
395	}	×
396	}
397
398	if err := scanner.Err(); err != nil {	×
399	return nil, fmt.Errorf("error reading patch: %w", err)	×
400	}	×
401
402	return &pbinternal.PrContents_File{	×
403	Name: filename,	×
404	FilePatchUrl: patchUrl,	×
405	PatchLines: result,	×
406	}, nil	×
407	}
408
409	func (di *Diff) getEcosystemForFile(filename string) DependencyEcosystem {	5✔
410	lastComponent := filepath.Base(filename)	5✔
411		5✔
412	for _, ecoMapping := range di.cfg.Ecosystems {	10✔
413	if match, _ := filepath.Match(ecoMapping.Depfile, lastComponent); match {	8✔
414	return DependencyEcosystem(ecoMapping.Name)	3✔
415	}	3✔
416	}
417	return DepEcosystemNone	2✔
418	}
419
420	func (di *Diff) getParserForFile(filename string, logger zerolog.Logger) ecosystemParser {	×
421	eco := di.getEcosystemForFile(filename)	×
422	if eco == DepEcosystemNone {	×
423	logger.Debug().	×
424	Str("filename", filename).	×
425	Msg("No ecosystem found, skipping")	×
426	return nil	×
427	}	×
428
429	logger.Debug().	×
430	Str("filename", filename).	×
431	Str("package-ecosystem", string(eco)).	×
432	Msg("matched ecosystem")	×
433		×
434	return newEcosystemParser(eco)	×
435	}
436
437	// Computes the set of elements in updated which are not in base.
438	// Note: this function may permute (sort) the order of elements in base and updated.
439	func setDifference[Slice ~[]E, E any](base Slice, updated Slice, sorter func(a, b E) int) Slice {	8✔
440		8✔
441	slices.SortFunc(base, sorter)	8✔
442	slices.SortFunc(updated, sorter)	8✔
443		8✔
444	baseIdx, newIdx := 0, 0	8✔
445	ret := make(Slice, 0)	8✔
446	for baseIdx < len(base) && newIdx < len(updated) {	25✔
447	cmpResult := sorter(base[baseIdx], updated[newIdx])	17✔
448	if cmpResult < 0 {	23✔
449	baseIdx++	6✔
450	} else if cmpResult > 0 {	22✔
451	ret = append(ret, updated[newIdx])	5✔
452	newIdx++	5✔
453	} else {	11✔
454	baseIdx++	6✔
455	newIdx++	6✔
456	}	6✔
457	}
458	if newIdx < len(updated) {	11✔
459	ret = append(ret, updated[newIdx:]...)	3✔
460	}	3✔
461
462	// TODO: add metric for number of deps scanned vs total deps
463
464	return ret	8✔
465	}

mindersec / minder / 29869011232

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous