• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 18791134616

24 Oct 2025 08:18PM UTC coverage: 75.519% (-4.8%) from 80.282%
18791134616

Pull #22794

github

web-flow
Merge 098c595a0 into 7971a20bf
Pull Request #22794: Use self-hosted MacOS Intel runner

65803 of 87134 relevant lines covered (75.52%)

3.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/src/python/pants/backend/go/goals/generate.py
1
# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
from __future__ import annotations
×
5

6
import os
×
7
import re
×
8
import shlex
×
9
import string
×
10
from collections.abc import Mapping
×
11
from dataclasses import dataclass
×
12

13
from pants.backend.go.target_types import GoPackageSourcesField
×
14
from pants.backend.go.util_rules import first_party_pkg, goroot, sdk
×
15
from pants.backend.go.util_rules.build_opts import (
×
16
    GoBuildOptionsFromTargetRequest,
17
    go_extract_build_options_from_target,
18
)
19
from pants.backend.go.util_rules.first_party_pkg import (
×
20
    FirstPartyPkgAnalysis,
21
    FirstPartyPkgAnalysisRequest,
22
    FirstPartyPkgDigestRequest,
23
    analyze_first_party_package,
24
    setup_first_party_pkg_digest,
25
)
26
from pants.backend.go.util_rules.goroot import GoRoot
×
27
from pants.build_graph.address import Address
×
28
from pants.core.util_rules.env_vars import environment_vars_subset
×
29
from pants.engine.env_vars import EnvironmentVarsRequest
×
30
from pants.engine.fs import CreateDigest, FileEntry, SnapshotDiff, Workspace
×
31
from pants.engine.goal import Goal, GoalSubsystem
×
32
from pants.engine.internals.native_engine import EMPTY_DIGEST, AddPrefix, Digest, MergeDigests
×
33
from pants.engine.internals.selectors import concurrently
×
34
from pants.engine.intrinsics import (
×
35
    add_prefix,
36
    create_digest,
37
    digest_to_snapshot,
38
    get_digest_contents,
39
    get_digest_entries,
40
    merge_digests,
41
)
42
from pants.engine.process import Process, execute_process_or_raise
×
43
from pants.engine.rules import collect_rules, goal_rule, implicitly, rule
×
44
from pants.engine.target import Targets
×
45
from pants.option.option_types import StrListOption
×
46
from pants.option.subsystem import Subsystem
×
47
from pants.util.dirutil import group_by_dir
×
48
from pants.util.strutil import help_text, softwrap
×
49

50
# Adapted from Go toolchain.
51
# See https://github.com/golang/go/blob/master/src/cmd/go/internal/generate/generate.go and
52
# https://github.com/golang/go/blob/cc1b20e8adf83865a1dbffa259c7a04ef0699b43/src/os/env.go#L16-L96
53
#
54
# Original copyright:
55
#   // Copyright 2011 The Go Authors. All rights reserved.
56
#   // Use of this source code is governed by a BSD-style
57
#   // license that can be found in the LICENSE file.
58

59

60
_GENERATE_DIRECTIVE_RE = re.compile(rb"^//go:generate[ \t](.*)$")
×
61

62

63
class GoGenerateGoalSubsystem(GoalSubsystem):
×
64
    name = "go-generate"
×
65
    help = help_text(
×
66
        """
67
        Run each command in a package described by a `//go:generate` directive. This is equivalent to running
68
        `go generate` on a Go package.
69

70
        Note: Just like with `go generate`, the `go-generate` goal is never run as part of the build and
71
        must be run manually to invoke the commands described by the `//go:generate` directives.
72

73
        See https://go.dev/blog/generate for details.
74
        """
75
    )
76

77
    class EnvironmentAware(Subsystem.EnvironmentAware):
×
78
        env_vars = StrListOption(
×
79
            default=["LANG", "LC_CTYPE", "LC_ALL", "PATH"],
80
            help=softwrap(
81
                """
82
                Environment variables to set when invoking generator programs.
83
                Entries are either strings in the form `ENV_VAR=value` to set an explicit value;
84
                or just `ENV_VAR` to copy the value from Pants's own environment.
85
                """
86
            ),
87
            advanced=True,
88
        )
89

90

91
class GoGenerateGoal(Goal):
×
92
    subsystem_cls = GoGenerateGoalSubsystem
×
93
    environment_behavior = Goal.EnvironmentBehavior.LOCAL_ONLY  # TODO(#17129) — Migrate this.
×
94

95

96
@dataclass(frozen=True)
×
97
class RunPackageGeneratorsRequest:
×
98
    address: Address
×
99
    regex: str | None = None
×
100

101

102
@dataclass(frozen=True)
×
103
class RunPackageGeneratorsResult:
×
104
    digest: Digest
×
105

106

107
_SHELL_SPECIAL_VAR = frozenset(
×
108
    ["*", "#", "$", "@", "!", "?", "-", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
109
)
110
_ALPHANUMERIC = frozenset([*string.ascii_letters, *string.digits, "_"])
×
111

112

113
def _get_shell_name(s: str) -> tuple[str, int]:
×
114
    if s[0] == "{":
×
115
        if len(s) > 2 and s[1] in _SHELL_SPECIAL_VAR and s[2] == "}":
×
116
            return s[1:2], 3
×
117
        for i in range(1, len(s)):
×
118
            if s[i] == "}":
×
119
                if i == 1:
×
120
                    return "", 2  # Bad syntax; eat "${}"
×
121
                return s[1:i], i + 1
×
122
        return "", 1  # Bad syntax; eat "${"
×
123
    elif s[0] in _SHELL_SPECIAL_VAR:
×
124
        return s[0:1], 1
×
125

126
    i = 0
×
127
    while i < len(s) and s[i] in _ALPHANUMERIC:
×
128
        i += 1
×
129

130
    return s[:i], i
×
131

132

133
def _expand_env(s: str, m: Mapping[str, str]) -> str:
×
134
    i = 0
×
135
    buf: str | None = None
×
136
    j = 0
×
137
    while j < len(s):
×
138
        if s[j] == "$" and j + 1 < len(s):
×
139
            if buf is None:
×
140
                buf = ""
×
141
            buf += s[i:j]
×
142
            name, w = _get_shell_name(s[j + 1 :])
×
143
            if name == "" and w > 0:
×
144
                # Encountered invalid syntax; eat the characters.
145
                pass
×
146
            elif name == "":
×
147
                # Valid syntax, but $ was not followed by a name. Leave the dollar character untouched.
148
                buf += s[j]
×
149
            else:
150
                buf += m.get(name, "")
×
151
            j += w
×
152
            i = j + 1
×
153
        j += 1
×
154

155
    if buf is None:
×
156
        return s
×
157
    return buf + s[i:]
×
158

159

160
async def _run_generators(
×
161
    analysis: FirstPartyPkgAnalysis,
162
    digest: Digest,
163
    dir_path: str,
164
    go_file: str,
165
    goroot: GoRoot,
166
    base_env: Mapping[str, str],
167
) -> Digest:
168
    digest_contents = await get_digest_contents(digest)
×
169
    content: bytes | None = None
×
170
    for entry in digest_contents:
×
171
        if entry.path == os.path.join(dir_path, go_file):
×
172
            content = entry.content
×
173
            break
×
174

175
    if content is None:
×
176
        raise ValueError("Illegal state: Unable to extract Go file from digest.")
×
177

178
    cmd_shorthand: dict[str, tuple[str, ...]] = {}
×
179

180
    for line_num, line in enumerate(content.splitlines(), start=1):
×
181
        m = _GENERATE_DIRECTIVE_RE.fullmatch(line)
×
182
        if not m:
×
183
            continue
×
184

185
        # Extract the command to run.
186
        # Note: Go only processes double-quoted strings. Thus, using shlex.split is actually more liberal than
187
        # Go because it also allows single-quoted strings.
188
        args = shlex.split(m.group(1).decode())
×
189

190
        # Store any command shorthands for later use.
191
        if args[0] == "-command":
×
192
            if len(args) <= 1:
×
193
                raise ValueError(
×
194
                    f"{go_file}:{line_num}: -command syntax used but no command name specified"
195
                )
196
            cmd_shorthand[args[1]] = tuple(args[2:])
×
197
            continue
×
198

199
        # Replace any shorthand command with the previously-stored arguments.
200
        if args[0] in cmd_shorthand:
×
201
            args = [*cmd_shorthand[args[0]], *args[1:]]
×
202

203
        # If the program calls for `go`, then use the full path to the `go` binary in the GOROOT.
204
        if args[0] == "go":
×
205
            args[0] = os.path.join(goroot.path, "bin", "go")
×
206

207
        env = {
×
208
            "GOOS": goroot.goos,
209
            "GOARCH": goroot.goarch,
210
            "GOFILE": go_file,
211
            "GOLINE": str(line_num),
212
            "GOPACKAGE": analysis.name,
213
            "GOROOT": goroot.path,
214
            "DOLLAR": "$",
215
            **base_env,
216
        }
217

218
        for i, arg in enumerate(args):
×
219
            args[i] = _expand_env(arg, env)
×
220

221
        # Invoke the subprocess and store its output for use as input root of next command (if any).
222
        result = await execute_process_or_raise(  # noqa: PNT30: this is inherently sequential
×
223
            **implicitly(
224
                Process(
225
                    argv=args,
226
                    input_digest=digest,
227
                    working_directory=dir_path,
228
                    output_directories=["."],
229
                    env=env,
230
                    description=f"Process `go generate` directives in file: {os.path.join(dir_path, go_file)}",
231
                ),
232
            ),
233
        )
234

235
        digest = await add_prefix(  # noqa: PNT30: this is inherently sequential
×
236
            AddPrefix(result.output_digest, dir_path)
237
        )
238

239
    return digest
×
240

241

242
@dataclass(frozen=True)
×
243
class OverwriteMergeDigests:
×
244
    orig_digest: Digest
×
245
    new_digest: Digest
×
246

247

248
@rule
×
249
async def merge_digests_with_overwrite(request: OverwriteMergeDigests) -> Digest:
×
250
    orig_snapshot, new_snapshot, orig_digest_entries, new_digest_entries = await concurrently(
×
251
        digest_to_snapshot(request.orig_digest),
252
        digest_to_snapshot(request.new_digest),
253
        get_digest_entries(request.orig_digest),
254
        get_digest_entries(request.new_digest),
255
    )
256

257
    orig_snapshot_grouped = group_by_dir(orig_snapshot.files)
×
258
    new_snapshot_grouped = group_by_dir(new_snapshot.files)
×
259

260
    diff = SnapshotDiff.from_snapshots(orig_snapshot, new_snapshot)
×
261

262
    output_entries: list[FileEntry] = []
×
263

264
    # Keep unchanged original files and directories in the output.
265
    orig_files_to_keep = set(diff.our_unique_files)
×
266
    for dir_path in diff.our_unique_dirs:
×
267
        for filename in orig_snapshot_grouped[dir_path]:
×
268
            orig_files_to_keep.add(os.path.join(dir_path, filename))
×
269
    for entry in orig_digest_entries:
×
270
        if isinstance(entry, FileEntry) and entry.path in orig_files_to_keep:
×
271
            output_entries.append(entry)
×
272

273
    # Add new files/directories and changed files to the output.
274
    new_files_to_keep = {*diff.their_unique_files, *diff.changed_files}
×
275
    for dir_path in diff.their_unique_dirs:
×
276
        for filename in new_snapshot_grouped[dir_path]:
×
277
            new_files_to_keep.add(os.path.join(dir_path, filename))
×
278
    for entry in new_digest_entries:
×
279
        if isinstance(entry, FileEntry) and entry.path in new_files_to_keep:
×
280
            output_entries.append(entry)
×
281

282
    digest = await create_digest(CreateDigest(output_entries))
×
283
    return digest
×
284

285

286
@rule
×
287
async def run_go_package_generators(
×
288
    request: RunPackageGeneratorsRequest,
289
    goroot: GoRoot,
290
    subsystem: GoGenerateGoalSubsystem.EnvironmentAware,
291
) -> RunPackageGeneratorsResult:
292
    build_opts = await go_extract_build_options_from_target(
×
293
        GoBuildOptionsFromTargetRequest(request.address), **implicitly()
294
    )
295
    fallible_analysis, env = await concurrently(
×
296
        analyze_first_party_package(
297
            FirstPartyPkgAnalysisRequest(
298
                request.address, build_opts=build_opts, extra_build_tags=("generate",)
299
            ),
300
            **implicitly(),
301
        ),
302
        environment_vars_subset(EnvironmentVarsRequest(subsystem.env_vars), **implicitly()),
303
    )
304
    if not fallible_analysis.analysis:
×
305
        raise ValueError(f"Analysis failure for {request.address}: {fallible_analysis.stderr}")
×
306
    analysis = fallible_analysis.analysis
×
307
    dir_path = analysis.dir_path if analysis.dir_path else "."
×
308

309
    fallible_pkg_digest = await setup_first_party_pkg_digest(
×
310
        FirstPartyPkgDigestRequest(request.address, build_opts=build_opts)
311
    )
312
    if fallible_pkg_digest.pkg_digest is None:
×
313
        raise ValueError(
×
314
            f"Unable to obtain digest for {request.address}: {fallible_pkg_digest.stderr}"
315
        )
316
    pkg_digest = fallible_pkg_digest.pkg_digest
×
317

318
    # Scan each Go file in the package for generate directives. Process them sequentially so that an error in
319
    # an earlier-processed file prevents later files from being processed.
320
    output_digest = EMPTY_DIGEST
×
321
    for go_file in analysis.go_files:
×
322
        output_digest_for_go_file = await _run_generators(
×
323
            analysis, pkg_digest.digest, dir_path, go_file, goroot, env
324
        )
325
        output_digest = await merge_digests_with_overwrite(  # noqa: PNT30: requires triage
×
326
            OverwriteMergeDigests(output_digest, output_digest_for_go_file)
327
        )
328

329
    return RunPackageGeneratorsResult(output_digest)
×
330

331

332
@goal_rule
×
333
async def go_generate(targets: Targets, workspace: Workspace) -> GoGenerateGoal:
×
334
    go_package_targets = [tgt for tgt in targets if tgt.has_field(GoPackageSourcesField)]
×
335
    results = await concurrently(
×
336
        run_go_package_generators(RunPackageGeneratorsRequest(tgt.address), **implicitly())
337
        for tgt in go_package_targets
338
    )
339
    output_digest = await merge_digests(MergeDigests([r.digest for r in results]))
×
340
    workspace.write_digest(output_digest)
×
341
    return GoGenerateGoal(exit_code=0)
×
342

343

344
def rules():
×
345
    return (
×
346
        *collect_rules(),
347
        *first_party_pkg.rules(),
348
        *goroot.rules(),
349
        *sdk.rules(),
350
    )
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc