• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 25441711719

06 May 2026 02:31PM UTC coverage: 92.915%. Remained the same
25441711719

push

github

web-flow
use sha pin (with comment) format for generated actions (#23312)

Per the GitHub Action best practices we recently enabled at #23249, we
should pin each action to a SHA so that the reference is actually
immutable.

This will -- I hope -- knock out a large chunk of the 421 alerts we
currently get from zizmor. The next followup would then be upgrades and
harmonizing the generated and none-generated pins.

Notice: This idea was suggested by Claude while going over pinact output
and I was surprised to see that post processing the yaml wasn't too
gross.

92206 of 99237 relevant lines covered (92.91%)

4.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.97
/src/python/pants/backend/go/goals/generate.py
1
# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
from __future__ import annotations
1✔
5

6
import os
1✔
7
import re
1✔
8
import shlex
1✔
9
import string
1✔
10
from collections.abc import Mapping
1✔
11
from dataclasses import dataclass
1✔
12

13
from pants.backend.go.target_types import GoPackageSourcesField
1✔
14
from pants.backend.go.util_rules import first_party_pkg, goroot, sdk
1✔
15
from pants.backend.go.util_rules.build_opts import (
1✔
16
    GoBuildOptionsFromTargetRequest,
17
    go_extract_build_options_from_target,
18
)
19
from pants.backend.go.util_rules.first_party_pkg import (
1✔
20
    FirstPartyPkgAnalysis,
21
    FirstPartyPkgAnalysisRequest,
22
    FirstPartyPkgDigestRequest,
23
    analyze_first_party_package,
24
    setup_first_party_pkg_digest,
25
)
26
from pants.backend.go.util_rules.goroot import GoRoot
1✔
27
from pants.build_graph.address import Address
1✔
28
from pants.core.util_rules.env_vars import environment_vars_subset
1✔
29
from pants.engine.env_vars import EnvironmentVarsRequest
1✔
30
from pants.engine.fs import CreateDigest, FileEntry, SnapshotDiff, Workspace
1✔
31
from pants.engine.goal import Goal, GoalSubsystem
1✔
32
from pants.engine.internals.native_engine import EMPTY_DIGEST, AddPrefix, Digest, MergeDigests
1✔
33
from pants.engine.internals.selectors import concurrently
1✔
34
from pants.engine.intrinsics import (
1✔
35
    add_prefix,
36
    create_digest,
37
    digest_to_snapshot,
38
    get_digest_contents,
39
    get_digest_entries,
40
    merge_digests,
41
)
42
from pants.engine.process import Process, execute_process_or_raise
1✔
43
from pants.engine.rules import collect_rules, goal_rule, implicitly, rule
1✔
44
from pants.engine.target import Targets
1✔
45
from pants.option.option_types import StrListOption
1✔
46
from pants.option.subsystem import Subsystem
1✔
47
from pants.util.dirutil import group_by_dir
1✔
48
from pants.util.strutil import help_text, softwrap
1✔
49

50
# Adapted from Go toolchain.
51
# See https://github.com/golang/go/blob/master/src/cmd/go/internal/generate/generate.go and
52
# https://github.com/golang/go/blob/cc1b20e8adf83865a1dbffa259c7a04ef0699b43/src/os/env.go#L16-L96
53
#
54
# Original copyright:
55
#   // Copyright 2011 The Go Authors. All rights reserved.
56
#   // Use of this source code is governed by a BSD-style
57
#   // license that can be found in the LICENSE file.
58

59

60
_GENERATE_DIRECTIVE_RE = re.compile(rb"^//go:generate[ \t](.*)$")
1✔
61

62

63
class GoGenerateGoalSubsystem(GoalSubsystem):
1✔
64
    name = "go-generate"
1✔
65
    help = help_text(
1✔
66
        """
67
        Run each command in a package described by a `//go:generate` directive. This is equivalent to running
68
        `go generate` on a Go package.
69

70
        Note: Just like with `go generate`, the `go-generate` goal is never run as part of the build and
71
        must be run manually to invoke the commands described by the `//go:generate` directives.
72

73
        See https://go.dev/blog/generate for details.
74
        """
75
    )
76

77
    class EnvironmentAware(Subsystem.EnvironmentAware):
1✔
78
        env_vars = StrListOption(
1✔
79
            default=["LANG", "LC_CTYPE", "LC_ALL", "PATH"],
80
            help=softwrap(
81
                """
82
                Environment variables to set when invoking generator programs.
83
                Entries are either strings in the form `ENV_VAR=value` to set an explicit value;
84
                or just `ENV_VAR` to copy the value from Pants's own environment.
85
                """
86
            ),
87
            advanced=True,
88
        )
89

90

91
class GoGenerateGoal(Goal):
1✔
92
    subsystem_cls = GoGenerateGoalSubsystem
1✔
93
    environment_behavior = Goal.EnvironmentBehavior.LOCAL_ONLY  # TODO(#17129) — Migrate this.
1✔
94

95

96
@dataclass(frozen=True)
1✔
97
class RunPackageGeneratorsRequest:
1✔
98
    address: Address
1✔
99
    regex: str | None = None
1✔
100

101

102
@dataclass(frozen=True)
1✔
103
class RunPackageGeneratorsResult:
1✔
104
    digest: Digest
1✔
105

106

107
_SHELL_SPECIAL_VAR = frozenset(
1✔
108
    ["*", "#", "$", "@", "!", "?", "-", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
109
)
110
_ALPHANUMERIC = frozenset([*string.ascii_letters, *string.digits, "_"])
1✔
111

112

113
def _get_shell_name(s: str) -> tuple[str, int]:
1✔
114
    if s[0] == "{":
1✔
115
        if len(s) > 2 and s[1] in _SHELL_SPECIAL_VAR and s[2] == "}":
1✔
116
            return s[1:2], 3
1✔
117
        for i in range(1, len(s)):
1✔
118
            if s[i] == "}":
1✔
119
                if i == 1:
1✔
120
                    return "", 2  # Bad syntax; eat "${}"
1✔
121
                return s[1:i], i + 1
1✔
122
        return "", 1  # Bad syntax; eat "${"
1✔
123
    elif s[0] in _SHELL_SPECIAL_VAR:
1✔
124
        return s[0:1], 1
1✔
125

126
    i = 0
1✔
127
    while i < len(s) and s[i] in _ALPHANUMERIC:
1✔
128
        i += 1
1✔
129

130
    return s[:i], i
1✔
131

132

133
def _expand_env(s: str, m: Mapping[str, str]) -> str:
1✔
134
    i = 0
1✔
135
    buf: str | None = None
1✔
136
    j = 0
1✔
137
    while j < len(s):
1✔
138
        if s[j] == "$" and j + 1 < len(s):
1✔
139
            if buf is None:
1✔
140
                buf = ""
1✔
141
            buf += s[i:j]
1✔
142
            name, w = _get_shell_name(s[j + 1 :])
1✔
143
            if name == "" and w > 0:
1✔
144
                # Encountered invalid syntax; eat the characters.
145
                pass
1✔
146
            elif name == "":
1✔
147
                # Valid syntax, but $ was not followed by a name. Leave the dollar character untouched.
148
                buf += s[j]
1✔
149
            else:
150
                buf += m.get(name, "")
1✔
151
            j += w
1✔
152
            i = j + 1
1✔
153
        j += 1
1✔
154

155
    if buf is None:
1✔
156
        return s
1✔
157
    return buf + s[i:]
1✔
158

159

160
async def _run_generators(
1✔
161
    analysis: FirstPartyPkgAnalysis,
162
    digest: Digest,
163
    dir_path: str,
164
    go_file: str,
165
    goroot: GoRoot,
166
    base_env: Mapping[str, str],
167
) -> Digest:
168
    digest_contents = await get_digest_contents(digest)
1✔
169
    content: bytes | None = None
1✔
170
    for entry in digest_contents:
1✔
171
        if entry.path == os.path.join(dir_path, go_file):
1✔
172
            content = entry.content
1✔
173
            break
1✔
174

175
    if content is None:
1✔
176
        raise ValueError("Illegal state: Unable to extract Go file from digest.")
×
177

178
    cmd_shorthand: dict[str, tuple[str, ...]] = {}
1✔
179

180
    for line_num, line in enumerate(content.splitlines(), start=1):
1✔
181
        m = _GENERATE_DIRECTIVE_RE.fullmatch(line)
1✔
182
        if not m:
1✔
183
            continue
1✔
184

185
        # Extract the command to run.
186
        # Note: Go only processes double-quoted strings. Thus, using shlex.split is actually more liberal than
187
        # Go because it also allows single-quoted strings.
188
        args = shlex.split(m.group(1).decode())
1✔
189

190
        # Store any command shorthands for later use.
191
        if args[0] == "-command":
1✔
192
            if len(args) <= 1:
1✔
193
                raise ValueError(
×
194
                    f"{go_file}:{line_num}: -command syntax used but no command name specified"
195
                )
196
            cmd_shorthand[args[1]] = tuple(args[2:])
1✔
197
            continue
1✔
198

199
        # Replace any shorthand command with the previously-stored arguments.
200
        if args[0] in cmd_shorthand:
1✔
201
            args = [*cmd_shorthand[args[0]], *args[1:]]
1✔
202

203
        # If the program calls for `go`, then use the full path to the `go` binary in the GOROOT.
204
        if args[0] == "go":
1✔
205
            args[0] = os.path.join(goroot.path, "bin", "go")
×
206

207
        env = {
1✔
208
            "GOOS": goroot.goos,
209
            "GOARCH": goroot.goarch,
210
            "GOFILE": go_file,
211
            "GOLINE": str(line_num),
212
            "GOPACKAGE": analysis.name,
213
            "GOROOT": goroot.path,
214
            "DOLLAR": "$",
215
            **base_env,
216
        }
217

218
        for i, arg in enumerate(args):
1✔
219
            args[i] = _expand_env(arg, env)
1✔
220

221
        # Invoke the subprocess and store its output for use as input root of next command (if any).
222
        result = await execute_process_or_raise(
1✔
223
            **implicitly(
224
                Process(
225
                    argv=args,
226
                    input_digest=digest,
227
                    working_directory=dir_path,
228
                    output_directories=["."],
229
                    env=env,
230
                    description=f"Process `go generate` directives in file: {os.path.join(dir_path, go_file)}",
231
                ),
232
            ),
233
        )
234

235
        digest = await add_prefix(AddPrefix(result.output_digest, dir_path))
1✔
236

237
    return digest
1✔
238

239

240
@dataclass(frozen=True)
1✔
241
class OverwriteMergeDigests:
1✔
242
    orig_digest: Digest
1✔
243
    new_digest: Digest
1✔
244

245

246
@rule
1✔
247
async def merge_digests_with_overwrite(request: OverwriteMergeDigests) -> Digest:
1✔
248
    orig_snapshot, new_snapshot, orig_digest_entries, new_digest_entries = await concurrently(
1✔
249
        digest_to_snapshot(request.orig_digest),
250
        digest_to_snapshot(request.new_digest),
251
        get_digest_entries(request.orig_digest),
252
        get_digest_entries(request.new_digest),
253
    )
254

255
    orig_snapshot_grouped = group_by_dir(orig_snapshot.files)
1✔
256
    new_snapshot_grouped = group_by_dir(new_snapshot.files)
1✔
257

258
    diff = SnapshotDiff.from_snapshots(orig_snapshot, new_snapshot)
1✔
259

260
    output_entries: list[FileEntry] = []
1✔
261

262
    # Keep unchanged original files and directories in the output.
263
    orig_files_to_keep = set(diff.our_unique_files)
1✔
264
    for dir_path in diff.our_unique_dirs:
1✔
265
        for filename in orig_snapshot_grouped[dir_path]:
1✔
266
            orig_files_to_keep.add(os.path.join(dir_path, filename))
1✔
267
    for entry in orig_digest_entries:
1✔
268
        if isinstance(entry, FileEntry) and entry.path in orig_files_to_keep:
1✔
269
            output_entries.append(entry)
1✔
270

271
    # Add new files/directories and changed files to the output.
272
    new_files_to_keep = {*diff.their_unique_files, *diff.changed_files}
1✔
273
    for dir_path in diff.their_unique_dirs:
1✔
274
        for filename in new_snapshot_grouped[dir_path]:
1✔
275
            new_files_to_keep.add(os.path.join(dir_path, filename))
1✔
276
    for entry in new_digest_entries:
1✔
277
        if isinstance(entry, FileEntry) and entry.path in new_files_to_keep:
1✔
278
            output_entries.append(entry)
1✔
279

280
    digest = await create_digest(CreateDigest(output_entries))
1✔
281
    return digest
1✔
282

283

284
@rule
1✔
285
async def run_go_package_generators(
1✔
286
    request: RunPackageGeneratorsRequest,
287
    goroot: GoRoot,
288
    subsystem: GoGenerateGoalSubsystem.EnvironmentAware,
289
) -> RunPackageGeneratorsResult:
290
    build_opts = await go_extract_build_options_from_target(
1✔
291
        GoBuildOptionsFromTargetRequest(request.address), **implicitly()
292
    )
293
    fallible_analysis, env = await concurrently(
1✔
294
        analyze_first_party_package(
295
            FirstPartyPkgAnalysisRequest(
296
                request.address, build_opts=build_opts, extra_build_tags=("generate",)
297
            ),
298
            **implicitly(),
299
        ),
300
        environment_vars_subset(EnvironmentVarsRequest(subsystem.env_vars), **implicitly()),
301
    )
302
    if not fallible_analysis.analysis:
1✔
303
        raise ValueError(f"Analysis failure for {request.address}: {fallible_analysis.stderr}")
×
304
    analysis = fallible_analysis.analysis
1✔
305
    dir_path = analysis.dir_path if analysis.dir_path else "."
1✔
306

307
    fallible_pkg_digest = await setup_first_party_pkg_digest(
1✔
308
        FirstPartyPkgDigestRequest(request.address, build_opts=build_opts)
309
    )
310
    if fallible_pkg_digest.pkg_digest is None:
1✔
311
        raise ValueError(
×
312
            f"Unable to obtain digest for {request.address}: {fallible_pkg_digest.stderr}"
313
        )
314
    pkg_digest = fallible_pkg_digest.pkg_digest
1✔
315

316
    # Scan each Go file in the package for generate directives. Process them sequentially so that an error in
317
    # an earlier-processed file prevents later files from being processed.
318
    output_digest = EMPTY_DIGEST
1✔
319
    for go_file in analysis.go_files:
1✔
320
        output_digest_for_go_file = await _run_generators(
1✔
321
            analysis, pkg_digest.digest, dir_path, go_file, goroot, env
322
        )
323
        output_digest = await merge_digests_with_overwrite(
1✔
324
            OverwriteMergeDigests(output_digest, output_digest_for_go_file)
325
        )
326

327
    return RunPackageGeneratorsResult(output_digest)
1✔
328

329

330
@goal_rule
1✔
331
async def go_generate(targets: Targets, workspace: Workspace) -> GoGenerateGoal:
1✔
332
    go_package_targets = [tgt for tgt in targets if tgt.has_field(GoPackageSourcesField)]
1✔
333
    results = await concurrently(
1✔
334
        run_go_package_generators(RunPackageGeneratorsRequest(tgt.address), **implicitly())
335
        for tgt in go_package_targets
336
    )
337
    output_digest = await merge_digests(MergeDigests([r.digest for r in results]))
1✔
338
    workspace.write_digest(output_digest)
1✔
339
    return GoGenerateGoal(exit_code=0)
1✔
340

341

342
def rules():
1✔
343
    return (
1✔
344
        *collect_rules(),
345
        *first_party_pkg.rules(),
346
        *goroot.rules(),
347
        *sdk.rules(),
348
    )
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc