• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 18252085855

05 Oct 2025 01:28AM UTC coverage: 80.263% (-0.002%) from 80.265%
18252085855

push

github

web-flow
Prepare 2.30.0.dev2 (#22728)

77226 of 96216 relevant lines covered (80.26%)

3.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.38
/src/python/pants/backend/go/util_rules/third_party_pkg.py
1
# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
from __future__ import annotations
10✔
5

6
import dataclasses
10✔
7
import difflib
10✔
8
import json
10✔
9
import logging
10✔
10
import os
10✔
11
from dataclasses import dataclass
10✔
12
from typing import Any
10✔
13

14
import ijson.backends.python as ijson
10✔
15

16
from pants.backend.go.go_sources.load_go_binary import LoadedGoBinaryRequest, setup_go_binary
10✔
17
from pants.backend.go.target_types import GoModTarget
10✔
18
from pants.backend.go.util_rules import pkg_analyzer
10✔
19
from pants.backend.go.util_rules.build_opts import GoBuildOptions
10✔
20
from pants.backend.go.util_rules.cgo import CGoCompilerFlags
10✔
21
from pants.backend.go.util_rules.embedcfg import EmbedConfig
10✔
22
from pants.backend.go.util_rules.pkg_analyzer import PackageAnalyzerSetup
10✔
23
from pants.backend.go.util_rules.sdk import GoSdkProcess
10✔
24
from pants.build_graph.address import Address
10✔
25
from pants.engine.engine_aware import EngineAwareParameter
10✔
26
from pants.engine.fs import (
10✔
27
    EMPTY_DIGEST,
28
    CreateDigest,
29
    Digest,
30
    DigestSubset,
31
    FileContent,
32
    GlobExpansionConjunction,
33
    GlobMatchErrorBehavior,
34
    MergeDigests,
35
    PathGlobs,
36
)
37
from pants.engine.intrinsics import (
10✔
38
    create_digest,
39
    digest_to_snapshot,
40
    execute_process,
41
    get_digest_contents,
42
    merge_digests,
43
)
44
from pants.engine.process import Process, fallible_to_exec_result_or_raise
10✔
45
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
10✔
46
from pants.util.dirutil import group_by_dir
10✔
47
from pants.util.frozendict import FrozenDict
10✔
48
from pants.util.logging import LogLevel
10✔
49
from pants.util.ordered_set import FrozenOrderedSet
10✔
50

51
logger = logging.getLogger(__name__)
10✔
52

53

54
class GoThirdPartyPkgError(Exception):
10✔
55
    pass
10✔
56

57

58
@dataclass(frozen=True)
10✔
59
class ThirdPartyPkgAnalysis:
10✔
60
    """All the info and files needed to build a third-party package.
61

62
    The digest only contains the files for the package, with all prefixes stripped.
63
    """
64

65
    import_path: str
10✔
66
    name: str
10✔
67

68
    digest: Digest
10✔
69
    dir_path: str
10✔
70

71
    # Note that we don't care about test-related metadata like `TestImports`, as we'll never run
72
    # tests directly on a third-party package.
73
    imports: tuple[str, ...]
10✔
74
    go_files: tuple[str, ...]
10✔
75
    cgo_files: tuple[str, ...]
10✔
76
    cgo_flags: CGoCompilerFlags
10✔
77

78
    c_files: tuple[str, ...]
10✔
79
    cxx_files: tuple[str, ...]
10✔
80
    m_files: tuple[str, ...]
10✔
81
    h_files: tuple[str, ...]
10✔
82
    f_files: tuple[str, ...]
10✔
83
    s_files: tuple[str, ...]
10✔
84

85
    syso_files: tuple[str, ...]
10✔
86

87
    minimum_go_version: str | None
10✔
88

89
    embed_patterns: tuple[str, ...]
10✔
90
    test_embed_patterns: tuple[str, ...]
10✔
91
    xtest_embed_patterns: tuple[str, ...]
10✔
92

93
    embed_config: EmbedConfig | None = None
10✔
94
    test_embed_config: EmbedConfig | None = None
10✔
95
    xtest_embed_config: EmbedConfig | None = None
10✔
96

97
    error: GoThirdPartyPkgError | None = None
10✔
98

99

100
@dataclass(frozen=True)
10✔
101
class ThirdPartyPkgAnalysisRequest(EngineAwareParameter):
10✔
102
    """Request the info and digest needed to build a third-party package.
103

104
    The package's module must be included in the input `go.mod`/`go.sum`.
105
    """
106

107
    import_path: str
10✔
108
    go_mod_address: Address
10✔
109
    go_mod_digest: Digest
10✔
110
    go_mod_path: str
10✔
111
    build_opts: GoBuildOptions
10✔
112

113
    def debug_hint(self) -> str:
10✔
114
        return f"{self.import_path} from {self.go_mod_path}"
×
115

116

117
@dataclass(frozen=True)
10✔
118
class AllThirdPartyPackages(FrozenDict[str, ThirdPartyPkgAnalysis]):
10✔
119
    """All the packages downloaded from a go.mod, along with a digest of the downloaded files.
120

121
    The digest has files in the format `gopath/pkg/mod`, which is what `GoSdkProcess` sets `GOPATH`
122
    to. This means that you can include the digest in a process and Go will properly consume it as
123
    the `GOPATH`.
124
    """
125

126
    digest: Digest
10✔
127
    import_paths_to_pkg_info: FrozenDict[str, ThirdPartyPkgAnalysis]
10✔
128

129

130
@dataclass(frozen=True)
10✔
131
class AllThirdPartyPackagesRequest:
10✔
132
    go_mod_address: Address
10✔
133
    go_mod_digest: Digest
10✔
134
    go_mod_path: str
10✔
135
    build_opts: GoBuildOptions
10✔
136

137

138
@dataclass(frozen=True)
10✔
139
class ModuleDescriptorsRequest:
10✔
140
    digest: Digest
10✔
141
    path: str
10✔
142

143

144
@dataclass(frozen=True)
10✔
145
class ModuleDescriptor:
10✔
146
    import_path: str
10✔
147
    name: str
10✔
148
    version: str
10✔
149
    indirect: bool
10✔
150
    minimum_go_version: str | None
10✔
151

152

153
@dataclass(frozen=True)
10✔
154
class ModuleDescriptors:
10✔
155
    modules: FrozenOrderedSet[ModuleDescriptor]
10✔
156
    go_mods_digest: Digest
10✔
157

158

159
@dataclass(frozen=True)
10✔
160
class AnalyzeThirdPartyModuleRequest:
10✔
161
    go_mod_address: Address
10✔
162
    go_mod_digest: Digest
10✔
163
    go_mod_path: str
10✔
164
    import_path: str
10✔
165
    name: str
10✔
166
    version: str
10✔
167
    minimum_go_version: str | None
10✔
168
    build_opts: GoBuildOptions
10✔
169

170

171
@dataclass(frozen=True)
10✔
172
class AnalyzedThirdPartyModule:
10✔
173
    packages: FrozenOrderedSet[ThirdPartyPkgAnalysis]
10✔
174

175

176
@dataclass(frozen=True)
10✔
177
class AnalyzeThirdPartyPackageRequest:
10✔
178
    pkg_json: FrozenDict[str, Any]
10✔
179
    module_sources_digest: Digest
10✔
180
    module_sources_path: str
10✔
181
    module_import_path: str
10✔
182
    package_path: str
10✔
183
    minimum_go_version: str | None
10✔
184

185

186
@dataclass(frozen=True)
10✔
187
class FallibleThirdPartyPkgAnalysis:
10✔
188
    """Metadata for a third-party Go package, but fallible if our analysis failed."""
189

190
    analysis: ThirdPartyPkgAnalysis | None
10✔
191
    import_path: str
10✔
192
    exit_code: int = 0
10✔
193
    stderr: str | None = None
10✔
194

195

196
@rule
10✔
197
async def analyze_module_dependencies(request: ModuleDescriptorsRequest) -> ModuleDescriptors:
10✔
198
    # List the modules used directly and indirectly by this module.
199
    #
200
    # This rule can't modify `go.mod` and `go.sum` as it would require mutating the workspace.
201
    # Instead, we expect them to be well-formed already.
202
    #
203
    # Options used:
204
    # - `-mod=readonly': It would be convenient to set `-mod=mod` to allow edits, and then compare the
205
    #   resulting files to the input so that we could print a diff for the user to know how to update. But
206
    #   `-mod=mod` results in more packages being downloaded and added to `go.mod` than is
207
    #   actually necessary.
208
    # TODO: nice error when `go.mod` and `go.sum` would need to change. Right now, it's a
209
    #  message from Go and won't be intuitive for Pants users what to do.
210
    # - `-e` is used to not fail if one of the modules is problematic. There may be some packages in the transitive
211
    #   closure that cannot be built, but we should  not blow up Pants. For example, a package that sets the
212
    #   special value `package documentation` and has no source files would naively error due to
213
    #   `build constraints exclude all Go files`, even though we should not error on that package.
214
    mod_list_result = await fallible_to_exec_result_or_raise(
×
215
        **implicitly(
216
            GoSdkProcess(
217
                command=["list", "-mod=readonly", "-e", "-m", "-json", "all"],
218
                input_digest=request.digest,
219
                output_directories=("gopath",),
220
                working_dir=request.path if request.path else None,
221
                # Allow downloads of the module metadata (i.e., go.mod files).
222
                allow_downloads=True,
223
                description="Analyze Go module dependencies.",
224
            )
225
        )
226
    )
227

228
    if len(mod_list_result.stdout) == 0:
×
229
        return ModuleDescriptors(FrozenOrderedSet(), EMPTY_DIGEST)
×
230

231
    descriptors: dict[tuple[str, str], ModuleDescriptor] = {}
×
232

233
    for mod_json in ijson.items(mod_list_result.stdout, "", multiple_values=True):
×
234
        # Skip the first-party module being analyzed.
235
        if "Main" in mod_json and mod_json["Main"]:
×
236
            continue
×
237

238
        # Skip first-party modules referenced from other first-party modules.
239
        # TODO Issue #22097: These cross-module references could be used for dependency inference
240
        if "Replace" in mod_json and "Version" not in mod_json["Replace"]:
×
241
            continue
×
242

243
        if "Replace" in mod_json:
×
244
            # TODO: Reject local file path replacements? Gazelle does.
245
            name = mod_json["Replace"]["Path"]
×
246
            version = mod_json["Replace"]["Version"]
×
247
        else:
248
            name = mod_json["Path"]
×
249
            version = mod_json["Version"]
×
250

251
        descriptors[(name, version)] = ModuleDescriptor(
×
252
            import_path=mod_json["Path"],
253
            name=name,
254
            version=version,
255
            indirect=mod_json.get("Indirect", False),
256
            minimum_go_version=mod_json.get("GoVersion"),
257
        )
258

259
    # TODO: Augment the modules with go.sum entries?
260
    # Gazelle does this, mainly to store the sum on the go_repository rule. We could store it (or its
261
    # absence) to be able to download sums automatically.
262

263
    return ModuleDescriptors(FrozenOrderedSet(descriptors.values()), mod_list_result.output_digest)
×
264

265

266
def strip_sandbox_prefix(path: str, marker: str) -> str:
10✔
267
    """Strip a path prefix from a path using a marker string to find the start of the portion to not
268
    strip. This is used to strip absolute paths used in the execution sandbox by `go`.
269

270
    Note: The marker string is required because we cannot assume how the prefix will be formed since it
271
    will differ depending on which execution environment is used (e.g, local or remote).
272
    """
273
    marker_pos = path.find(marker)
×
274
    if marker_pos != -1:
×
275
        return path[marker_pos:]
×
276
    else:
277
        return path
×
278

279

280
def _freeze_json_dict(d: dict[Any, Any]) -> FrozenDict[str, Any]:
10✔
281
    result = {}
×
282
    for k, v in d.items():
×
283
        if not isinstance(k, str):
×
284
            raise AssertionError("Got non-`str` key for _freeze_json_dict.")
×
285

286
        f: Any = None
×
287
        if isinstance(v, list):
×
288
            f = tuple(v)
×
289
        elif isinstance(v, dict):
×
290
            f = _freeze_json_dict(v)
×
291
        elif isinstance(v, str) or isinstance(v, int):
×
292
            f = v
×
293
        else:
294
            raise AssertionError(f"Unsupported value type for _freeze_json_dict: {type(v)}")
×
295
        result[k] = f
×
296
    return FrozenDict(result)
×
297

298

299
async def _check_go_sum_has_not_changed(
10✔
300
    input_digest: Digest,
301
    output_digest: Digest,
302
    dir_path: str,
303
    import_path: str,
304
    go_mod_address: Address,
305
) -> None:
306
    input_entries, output_entries = await concurrently(
×
307
        get_digest_contents(input_digest),
308
        get_digest_contents(output_digest),
309
    )
310

311
    go_sum_path = os.path.join(dir_path, "go.sum")
×
312

313
    input_go_sum_entry: bytes | None = None
×
314
    for entry in input_entries:
×
315
        if entry.path == go_sum_path:
×
316
            input_go_sum_entry = entry.content
×
317

318
    output_go_sum_entry: bytes | None = None
×
319
    for entry in output_entries:
×
320
        if entry.path == go_sum_path:
×
321
            output_go_sum_entry = entry.content
×
322

323
    if input_go_sum_entry is not None or output_go_sum_entry is not None:
×
324
        if input_go_sum_entry != output_go_sum_entry:
×
325
            go_sum_diff = list(
×
326
                difflib.unified_diff(
327
                    (input_go_sum_entry or b"").decode().splitlines(),
328
                    (output_go_sum_entry or b"").decode().splitlines(),
329
                )
330
            )
331
            go_sum_diff_rendered = "\n".join(line.rstrip() for line in go_sum_diff)
×
332
            raise ValueError(
×
333
                f"For `{GoModTarget.alias}` target `{go_mod_address}`, the go.sum file is incomplete "
334
                f"because it was updated while processing third-party dependency `{import_path}`. "
335
                "Please re-generate the go.sum file by running `go mod download all` in the module directory. "
336
                "(Pants does not currently have support for updating the go.sum checksum database itself.)\n\n"
337
                f"Diff:\n{go_sum_diff_rendered}"
338
            )
339

340

341
@rule
10✔
342
async def analyze_go_third_party_package(
10✔
343
    request: AnalyzeThirdPartyPackageRequest,
344
) -> FallibleThirdPartyPkgAnalysis:
345
    if not request.package_path.startswith(request.module_sources_path):
×
346
        raise AssertionError(
×
347
            "The path within GOPATH for a package in a module must always be prefixed by the path "
348
            "to the applicable module's root directory. "
349
            f"This was not the case however for module {request.module_import_path}.\n\n"
350
            "This may be a bug in Pants. Please report this issue at "
351
            "https://github.com/pantsbuild/pants/issues/new/choose and include the following data: "
352
            f"package_path: {request.package_path}; module_sources_path: {request.module_sources_path}; "
353
            f"module_import_path: {request.module_import_path}"
354
        )
355
    import_path_tail = request.package_path[len(request.module_sources_path) :].strip(os.sep)
×
356
    if import_path_tail != "":
×
357
        parts = import_path_tail.split(os.sep)
×
358
        import_path = "/".join([request.module_import_path, *parts])
×
359
    else:
360
        import_path = request.module_import_path
×
361

362
    if "Error" in request.pkg_json or "InvalidGoFiles" in request.pkg_json:
×
363
        error = request.pkg_json.get("Error", "")
×
364
        if error:
×
365
            error += "\n"
×
366
        if "InvalidGoFiles" in request.pkg_json:
×
367
            error += "\n".join(
×
368
                f"{filename}: {error}"
369
                for filename, error in request.pkg_json.get("InvalidGoFiles", {}).items()
370
            )
371
            error += "\n"
×
372
        return FallibleThirdPartyPkgAnalysis(
×
373
            analysis=None, import_path=import_path, exit_code=1, stderr=error
374
        )
375

376
    maybe_error: GoThirdPartyPkgError | None = None
×
377

378
    for key in (
×
379
        "CompiledGoFiles",
380
        "SwigFiles",
381
        "SwigCXXFiles",
382
    ):
383
        if key in request.pkg_json:
×
384
            maybe_error = GoThirdPartyPkgError(
×
385
                f"The third-party package {import_path} includes `{key}`, which Pants does "
386
                "not yet support. Please open a feature request at "
387
                "https://github.com/pantsbuild/pants/issues/new/choose so that we know to "
388
                "prioritize adding support. Please include this error message and the version of "
389
                "the third-party module."
390
            )
391

392
    analysis = ThirdPartyPkgAnalysis(
×
393
        digest=request.module_sources_digest,
394
        import_path=import_path,
395
        name=request.pkg_json["Name"],
396
        dir_path=request.package_path,
397
        imports=tuple(request.pkg_json.get("Imports", ())),
398
        go_files=tuple(request.pkg_json.get("GoFiles", ())),
399
        c_files=tuple(request.pkg_json.get("CFiles", ())),
400
        cxx_files=tuple(request.pkg_json.get("CXXFiles", ())),
401
        m_files=tuple(request.pkg_json.get("MFiles", ())),
402
        h_files=tuple(request.pkg_json.get("HFiles", ())),
403
        f_files=tuple(request.pkg_json.get("FFiles", ())),
404
        s_files=tuple(request.pkg_json.get("SFiles", ())),
405
        syso_files=tuple(request.pkg_json.get("SysoFiles", ())),
406
        cgo_files=tuple(request.pkg_json.get("CgoFiles", ())),
407
        minimum_go_version=request.minimum_go_version,
408
        embed_patterns=tuple(request.pkg_json.get("EmbedPatterns", [])),
409
        test_embed_patterns=tuple(request.pkg_json.get("TestEmbedPatterns", [])),
410
        xtest_embed_patterns=tuple(request.pkg_json.get("XTestEmbedPatterns", [])),
411
        error=maybe_error,
412
        cgo_flags=CGoCompilerFlags(
413
            cflags=tuple(request.pkg_json.get("CgoCFLAGS", [])),
414
            cppflags=tuple(request.pkg_json.get("CgoCPPFLAGS", [])),
415
            cxxflags=tuple(request.pkg_json.get("CgoCXXFLAGS", [])),
416
            fflags=tuple(request.pkg_json.get("CgoFFLAGS", [])),
417
            ldflags=tuple(request.pkg_json.get("CgoLDFLAGS", [])),
418
            pkg_config=tuple(request.pkg_json.get("CgoPkgConfig", [])),
419
        ),
420
    )
421

422
    if analysis.embed_patterns or analysis.test_embed_patterns or analysis.xtest_embed_patterns:
×
423
        patterns_json = json.dumps(
×
424
            {
425
                "EmbedPatterns": analysis.embed_patterns,
426
                "TestEmbedPatterns": analysis.test_embed_patterns,
427
                "XTestEmbedPatterns": analysis.xtest_embed_patterns,
428
            }
429
        ).encode("utf-8")
430
        embedder, patterns_json_digest = await concurrently(
×
431
            setup_go_binary(
432
                LoadedGoBinaryRequest("embedcfg", ("main.go",), "./embedder"), **implicitly()
433
            ),
434
            create_digest(CreateDigest([FileContent("patterns.json", patterns_json)])),
435
        )
436
        input_digest = await merge_digests(
×
437
            MergeDigests((request.module_sources_digest, patterns_json_digest, embedder.digest))
438
        )
439
        embed_result = await execute_process(
×
440
            Process(
441
                ("./embedder", "patterns.json", request.package_path),
442
                input_digest=input_digest,
443
                description=f"Create embed mapping for {import_path}",
444
                level=LogLevel.DEBUG,
445
            ),
446
            **implicitly(),
447
        )
448
        if embed_result.exit_code != 0:
×
449
            return FallibleThirdPartyPkgAnalysis(
×
450
                analysis=None,
451
                import_path=import_path,
452
                exit_code=1,
453
                stderr=embed_result.stderr.decode(),
454
            )
455
        metadata = json.loads(embed_result.stdout)
×
456
        embed_config = EmbedConfig.from_json_dict(metadata.get("EmbedConfig", {}))
×
457
        test_embed_config = EmbedConfig.from_json_dict(metadata.get("TestEmbedConfig", {}))
×
458
        xtest_embed_config = EmbedConfig.from_json_dict(metadata.get("XTestEmbedConfig", {}))
×
459
        analysis = dataclasses.replace(
×
460
            analysis,
461
            embed_config=embed_config,
462
            test_embed_config=test_embed_config,
463
            xtest_embed_config=xtest_embed_config,
464
        )
465

466
    return FallibleThirdPartyPkgAnalysis(
×
467
        analysis=analysis,
468
        import_path=import_path,
469
        exit_code=0,
470
        stderr=None,
471
    )
472

473

474
@rule
10✔
475
async def analyze_go_third_party_module(
10✔
476
    request: AnalyzeThirdPartyModuleRequest,
477
    analyzer: PackageAnalyzerSetup,
478
) -> AnalyzedThirdPartyModule:
479
    # Download the module.
480
    download_result = await fallible_to_exec_result_or_raise(
×
481
        **implicitly(
482
            GoSdkProcess(
483
                ("mod", "download", "-json", f"{request.name}@{request.version}"),
484
                input_digest=request.go_mod_digest,  # for go.sum
485
                working_dir=os.path.dirname(request.go_mod_path),
486
                # Allow downloads of the module sources.
487
                allow_downloads=True,
488
                output_directories=("gopath",),
489
                output_files=(os.path.join(os.path.dirname(request.go_mod_path), "go.sum"),),
490
                description=f"Download Go module {request.name}@{request.version}.",
491
            )
492
        )
493
    )
494

495
    if len(download_result.stdout) == 0:
×
496
        raise AssertionError(
×
497
            f"Expected output from `go mod download` for {request.name}@{request.version}."
498
        )
499

500
    # Make sure go.sum has not changed.
501
    await _check_go_sum_has_not_changed(
×
502
        input_digest=request.go_mod_digest,
503
        output_digest=download_result.output_digest,
504
        dir_path=os.path.dirname(request.go_mod_path),
505
        import_path=request.import_path,
506
        go_mod_address=request.go_mod_address,
507
    )
508

509
    module_metadata = json.loads(download_result.stdout)
×
510
    module_sources_relpath = strip_sandbox_prefix(module_metadata["Dir"], "gopath/")
×
511
    go_mod_relpath = strip_sandbox_prefix(module_metadata["GoMod"], "gopath/")
×
512

513
    # Subset the output directory to just the module sources and go.mod (which may be generated).
514
    module_sources_snapshot = await digest_to_snapshot(
×
515
        **implicitly(
516
            DigestSubset(
517
                download_result.output_digest,
518
                PathGlobs(
519
                    [f"{module_sources_relpath}/**", go_mod_relpath],
520
                    glob_match_error_behavior=GlobMatchErrorBehavior.error,
521
                    conjunction=GlobExpansionConjunction.all_match,
522
                    description_of_origin=f"the download of Go module {request.name}@{request.version}",
523
                ),
524
            )
525
        )
526
    )
527

528
    # Determine directories with potential Go packages in them.
529
    candidate_package_dirs = []
×
530
    files_by_dir = group_by_dir(
×
531
        p for p in module_sources_snapshot.files if p.startswith(module_sources_relpath)
532
    )
533
    for maybe_pkg_dir, files in files_by_dir.items():
×
534
        # Skip directories where "testdata" would end up in the import path.
535
        # See https://github.com/golang/go/blob/f005df8b582658d54e63d59953201299d6fee880/src/go/build/build.go#L580-L585
536
        if "testdata" in maybe_pkg_dir.split("/"):
×
537
            continue
×
538

539
        # Consider directories with at least one `.go` file as package candidates.
540
        if any(f for f in files if f.endswith(".go")):
×
541
            candidate_package_dirs.append(maybe_pkg_dir)
×
542
    candidate_package_dirs.sort()
×
543

544
    # Analyze all of the packages in this module.
545
    analyzer_relpath = "__analyzer"
×
546
    analysis_result = await fallible_to_exec_result_or_raise(
×
547
        **implicitly(
548
            Process(
549
                [os.path.join(analyzer_relpath, analyzer.path), *candidate_package_dirs],
550
                input_digest=module_sources_snapshot.digest,
551
                immutable_input_digests={
552
                    analyzer_relpath: analyzer.digest,
553
                },
554
                description=f"Analyze metadata for Go third-party module: {request.name}@{request.version}",
555
                level=LogLevel.DEBUG,
556
                env={"CGO_ENABLED": "1" if request.build_opts.cgo_enabled else "0"},
557
            )
558
        )
559
    )
560

561
    if len(analysis_result.stdout) == 0:
×
562
        return AnalyzedThirdPartyModule(FrozenOrderedSet())
×
563

564
    package_analysis_gets = []
×
565
    for pkg_path, pkg_json in zip(
×
566
        candidate_package_dirs, ijson.items(analysis_result.stdout, "", multiple_values=True)
567
    ):
568
        package_analysis_gets.append(
×
569
            analyze_go_third_party_package(
570
                AnalyzeThirdPartyPackageRequest(
571
                    pkg_json=_freeze_json_dict(pkg_json),
572
                    module_sources_digest=module_sources_snapshot.digest,
573
                    module_sources_path=module_sources_relpath,
574
                    module_import_path=request.name,
575
                    package_path=pkg_path,
576
                    minimum_go_version=request.minimum_go_version,
577
                )
578
            )
579
        )
580
    analyzed_packages_fallible = await concurrently(package_analysis_gets)
×
581
    analyzed_packages = [
×
582
        pkg.analysis for pkg in analyzed_packages_fallible if pkg.analysis and pkg.exit_code == 0
583
    ]
584
    return AnalyzedThirdPartyModule(FrozenOrderedSet(analyzed_packages))
×
585

586

587
@rule(desc="Download and analyze all third-party Go packages", level=LogLevel.DEBUG)
10✔
588
async def download_and_analyze_third_party_packages(
10✔
589
    request: AllThirdPartyPackagesRequest,
590
) -> AllThirdPartyPackages:
591
    module_analysis = await analyze_module_dependencies(
×
592
        ModuleDescriptorsRequest(
593
            digest=request.go_mod_digest,
594
            path=os.path.dirname(request.go_mod_path),
595
        )
596
    )
597

598
    analyzed_modules = await concurrently(
×
599
        analyze_go_third_party_module(
600
            AnalyzeThirdPartyModuleRequest(
601
                go_mod_address=request.go_mod_address,
602
                go_mod_digest=request.go_mod_digest,
603
                go_mod_path=request.go_mod_path,
604
                import_path=mod.name,
605
                name=mod.name,
606
                version=mod.version,
607
                minimum_go_version=mod.minimum_go_version,
608
                build_opts=request.build_opts,
609
            ),
610
            **implicitly(),
611
        )
612
        for mod in module_analysis.modules
613
    )
614

615
    import_path_to_info = {
×
616
        pkg.import_path: pkg
617
        for analyzed_module in analyzed_modules
618
        for pkg in analyzed_module.packages
619
    }
620

621
    return AllThirdPartyPackages(EMPTY_DIGEST, FrozenDict(import_path_to_info))
×
622

623

624
@rule
10✔
625
async def extract_package_info(request: ThirdPartyPkgAnalysisRequest) -> ThirdPartyPkgAnalysis:
10✔
626
    all_packages = await download_and_analyze_third_party_packages(
×
627
        AllThirdPartyPackagesRequest(
628
            request.go_mod_address,
629
            request.go_mod_digest,
630
            request.go_mod_path,
631
            build_opts=request.build_opts,
632
        )
633
    )
634
    pkg_info = all_packages.import_paths_to_pkg_info.get(request.import_path)
×
635
    if pkg_info:
×
636
        return pkg_info
×
637
    raise AssertionError(
×
638
        f"The package `{request.import_path}` was not downloaded, but Pants tried using it. "
639
        "This should not happen. Please open an issue at "
640
        "https://github.com/pantsbuild/pants/issues/new/choose with this error message."
641
    )
642

643

644
def maybe_raise_or_create_error_or_create_failed_pkg_info(
10✔
645
    go_list_json: dict, import_path: str
646
) -> tuple[GoThirdPartyPkgError | None, ThirdPartyPkgAnalysis | None]:
647
    """Error for unrecoverable errors, otherwise lazily create an error or `ThirdPartyPkgInfo` for
648
    recoverable errors.
649

650
    Lazy errors should only be raised when the package is compiled, but not during target generation
651
    and project introspection. This is important so that we don't overzealously error on packages
652
    that the user doesn't actually ever use, given how a Go module includes all of its packages,
653
    even test packages that are never used by first-party code.
654

655
    Returns a `ThirdPartyPkgInfo` if the `Dir` key is missing, which is necessary for our normal
656
    analysis of the package.
657
    """
658
    if import_path == "...":
×
659
        if "Error" not in go_list_json:
×
660
            raise AssertionError(
×
661
                "`go list` included the import path `...`, but there was no `Error` attached. "
662
                "Please open an issue at https://github.com/pantsbuild/pants/issues/new/choose "
663
                f"with this error message:\n\n{go_list_json}"
664
            )
665
        # TODO: Improve this error message, such as better instructions if `go.sum` is stale.
666
        raise GoThirdPartyPkgError(go_list_json["Error"]["Err"])
×
667

668
    if "Dir" not in go_list_json:
×
669
        error = GoThirdPartyPkgError(
×
670
            f"`go list` failed for the import path `{import_path}` because `Dir` was not defined. "
671
            f"Please open an issue at https://github.com/pantsbuild/pants/issues/new/choose so "
672
            f"that we can figure out how to support this:"
673
            f"\n\n{go_list_json}"
674
        )
675
        return None, ThirdPartyPkgAnalysis(
×
676
            import_path=import_path,
677
            name="",
678
            dir_path="",
679
            digest=EMPTY_DIGEST,
680
            imports=(),
681
            go_files=(),
682
            c_files=(),
683
            cxx_files=(),
684
            h_files=(),
685
            m_files=(),
686
            f_files=(),
687
            s_files=(),
688
            syso_files=(),
689
            minimum_go_version=None,
690
            embed_patterns=(),
691
            test_embed_patterns=(),
692
            xtest_embed_patterns=(),
693
            error=error,
694
            cgo_files=(),
695
            cgo_flags=CGoCompilerFlags(
696
                cflags=(),
697
                cppflags=(),
698
                cxxflags=(),
699
                fflags=(),
700
                ldflags=(),
701
                pkg_config=(),
702
            ),
703
        )
704

705
    if "Error" in go_list_json:
×
706
        err_msg = go_list_json["Error"]["Err"]
×
707
        return (
×
708
            GoThirdPartyPkgError(
709
                f"`go list` failed for the import path `{import_path}`. Please open an issue at "
710
                "https://github.com/pantsbuild/pants/issues/new/choose so that we can figure out "
711
                "how to support this:"
712
                f"\n\n{err_msg}\n\n{go_list_json}"
713
            ),
714
            None,
715
        )
716

717
    return None, None
×
718

719

720
def rules():
10✔
721
    return (
10✔
722
        *collect_rules(),
723
        *pkg_analyzer.rules(),
724
    )
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc