• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 23074067894

13 Mar 2026 11:06PM UTC coverage: 64.165% (-28.8%) from 92.932%
23074067894

Pull #23171

github

web-flow
Merge 17d8ea7d8 into f07276df6
Pull Request #23171: Debug reapi test cache misses

42163 of 65710 relevant lines covered (64.17%)

0.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/src/python/pants/backend/go/goals/generate.py
1
# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
from __future__ import annotations
×
5

6
import os
×
7
import re
×
8
import shlex
×
9
import string
×
10
from collections.abc import Mapping
×
11
from dataclasses import dataclass
×
12

13
from pants.backend.go.target_types import GoPackageSourcesField
×
14
from pants.backend.go.util_rules import first_party_pkg, goroot, sdk
×
15
from pants.backend.go.util_rules.build_opts import (
×
16
    GoBuildOptionsFromTargetRequest,
17
    go_extract_build_options_from_target,
18
)
19
from pants.backend.go.util_rules.first_party_pkg import (
×
20
    FirstPartyPkgAnalysis,
21
    FirstPartyPkgAnalysisRequest,
22
    FirstPartyPkgDigestRequest,
23
    analyze_first_party_package,
24
    setup_first_party_pkg_digest,
25
)
26
from pants.backend.go.util_rules.goroot import GoRoot
×
27
from pants.build_graph.address import Address
×
28
from pants.core.util_rules.env_vars import environment_vars_subset
×
29
from pants.engine.env_vars import EnvironmentVarsRequest
×
30
from pants.engine.fs import CreateDigest, FileEntry, SnapshotDiff, Workspace
×
31
from pants.engine.goal import Goal, GoalSubsystem
×
32
from pants.engine.internals.native_engine import EMPTY_DIGEST, AddPrefix, Digest, MergeDigests
×
33
from pants.engine.internals.selectors import concurrently
×
34
from pants.engine.intrinsics import (
×
35
    add_prefix,
36
    create_digest,
37
    digest_to_snapshot,
38
    get_digest_contents,
39
    get_digest_entries,
40
    merge_digests,
41
)
42
from pants.engine.process import Process, execute_process_or_raise
×
43
from pants.engine.rules import collect_rules, goal_rule, implicitly, rule
×
44
from pants.engine.target import Targets
×
45
from pants.option.option_types import StrListOption
×
46
from pants.option.subsystem import Subsystem
×
47
from pants.util.dirutil import group_by_dir
×
48
from pants.util.strutil import help_text, softwrap
×
49

50
# Adapted from Go toolchain.
51
# See https://github.com/golang/go/blob/master/src/cmd/go/internal/generate/generate.go and
52
# https://github.com/golang/go/blob/cc1b20e8adf83865a1dbffa259c7a04ef0699b43/src/os/env.go#L16-L96
53
#
54
# Original copyright:
55
#   // Copyright 2011 The Go Authors. All rights reserved.
56
#   // Use of this source code is governed by a BSD-style
57
#   // license that can be found in the LICENSE file.
58

59

60
_GENERATE_DIRECTIVE_RE = re.compile(rb"^//go:generate[ \t](.*)$")
×
61

62

63
class GoGenerateGoalSubsystem(GoalSubsystem):
×
64
    name = "go-generate"
×
65
    help = help_text(
×
66
        """
67
        Run each command in a package described by a `//go:generate` directive. This is equivalent to running
68
        `go generate` on a Go package.
69

70
        Note: Just like with `go generate`, the `go-generate` goal is never run as part of the build and
71
        must be run manually to invoke the commands described by the `//go:generate` directives.
72

73
        See https://go.dev/blog/generate for details.
74
        """
75
    )
76

77
    class EnvironmentAware(Subsystem.EnvironmentAware):
×
78
        env_vars = StrListOption(
×
79
            default=["LANG", "LC_CTYPE", "LC_ALL", "PATH"],
80
            help=softwrap(
81
                """
82
                Environment variables to set when invoking generator programs.
83
                Entries are either strings in the form `ENV_VAR=value` to set an explicit value;
84
                or just `ENV_VAR` to copy the value from Pants's own environment.
85
                """
86
            ),
87
            advanced=True,
88
        )
89

90

91
class GoGenerateGoal(Goal):
×
92
    subsystem_cls = GoGenerateGoalSubsystem
×
93
    environment_behavior = Goal.EnvironmentBehavior.LOCAL_ONLY  # TODO(#17129) — Migrate this.
×
94

95

96
@dataclass(frozen=True)
×
97
class RunPackageGeneratorsRequest:
×
98
    address: Address
×
99
    regex: str | None = None
×
100

101

102
@dataclass(frozen=True)
×
103
class RunPackageGeneratorsResult:
×
104
    digest: Digest
×
105

106

107
_SHELL_SPECIAL_VAR = frozenset(
×
108
    ["*", "#", "$", "@", "!", "?", "-", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
109
)
110
_ALPHANUMERIC = frozenset([*string.ascii_letters, *string.digits, "_"])
×
111

112

113
def _get_shell_name(s: str) -> tuple[str, int]:
×
114
    if s[0] == "{":
×
115
        if len(s) > 2 and s[1] in _SHELL_SPECIAL_VAR and s[2] == "}":
×
116
            return s[1:2], 3
×
117
        for i in range(1, len(s)):
×
118
            if s[i] == "}":
×
119
                if i == 1:
×
120
                    return "", 2  # Bad syntax; eat "${}"
×
121
                return s[1:i], i + 1
×
122
        return "", 1  # Bad syntax; eat "${"
×
123
    elif s[0] in _SHELL_SPECIAL_VAR:
×
124
        return s[0:1], 1
×
125

126
    i = 0
×
127
    while i < len(s) and s[i] in _ALPHANUMERIC:
×
128
        i += 1
×
129

130
    return s[:i], i
×
131

132

133
def _expand_env(s: str, m: Mapping[str, str]) -> str:
×
134
    i = 0
×
135
    buf: str | None = None
×
136
    j = 0
×
137
    while j < len(s):
×
138
        if s[j] == "$" and j + 1 < len(s):
×
139
            if buf is None:
×
140
                buf = ""
×
141
            buf += s[i:j]
×
142
            name, w = _get_shell_name(s[j + 1 :])
×
143
            if name == "" and w > 0:
×
144
                # Encountered invalid syntax; eat the characters.
145
                pass
×
146
            elif name == "":
×
147
                # Valid syntax, but $ was not followed by a name. Leave the dollar character untouched.
148
                buf += s[j]
×
149
            else:
150
                buf += m.get(name, "")
×
151
            j += w
×
152
            i = j + 1
×
153
        j += 1
×
154

155
    if buf is None:
×
156
        return s
×
157
    return buf + s[i:]
×
158

159

160
async def _run_generators(
×
161
    analysis: FirstPartyPkgAnalysis,
162
    digest: Digest,
163
    dir_path: str,
164
    go_file: str,
165
    goroot: GoRoot,
166
    base_env: Mapping[str, str],
167
) -> Digest:
168
    digest_contents = await get_digest_contents(digest)
×
169
    content: bytes | None = None
×
170
    for entry in digest_contents:
×
171
        if entry.path == os.path.join(dir_path, go_file):
×
172
            content = entry.content
×
173
            break
×
174

175
    if content is None:
×
176
        raise ValueError("Illegal state: Unable to extract Go file from digest.")
×
177

178
    cmd_shorthand: dict[str, tuple[str, ...]] = {}
×
179

180
    for line_num, line in enumerate(content.splitlines(), start=1):
×
181
        m = _GENERATE_DIRECTIVE_RE.fullmatch(line)
×
182
        if not m:
×
183
            continue
×
184

185
        # Extract the command to run.
186
        # Note: Go only processes double-quoted strings. Thus, using shlex.split is actually more liberal than
187
        # Go because it also allows single-quoted strings.
188
        args = shlex.split(m.group(1).decode())
×
189

190
        # Store any command shorthands for later use.
191
        if args[0] == "-command":
×
192
            if len(args) <= 1:
×
193
                raise ValueError(
×
194
                    f"{go_file}:{line_num}: -command syntax used but no command name specified"
195
                )
196
            cmd_shorthand[args[1]] = tuple(args[2:])
×
197
            continue
×
198

199
        # Replace any shorthand command with the previously-stored arguments.
200
        if args[0] in cmd_shorthand:
×
201
            args = [*cmd_shorthand[args[0]], *args[1:]]
×
202

203
        # If the program calls for `go`, then use the full path to the `go` binary in the GOROOT.
204
        if args[0] == "go":
×
205
            args[0] = os.path.join(goroot.path, "bin", "go")
×
206

207
        env = {
×
208
            "GOOS": goroot.goos,
209
            "GOARCH": goroot.goarch,
210
            "GOFILE": go_file,
211
            "GOLINE": str(line_num),
212
            "GOPACKAGE": analysis.name,
213
            "GOROOT": goroot.path,
214
            "DOLLAR": "$",
215
            **base_env,
216
        }
217

218
        for i, arg in enumerate(args):
×
219
            args[i] = _expand_env(arg, env)
×
220

221
        # Invoke the subprocess and store its output for use as input root of next command (if any).
222
        result = await execute_process_or_raise(
×
223
            **implicitly(
224
                Process(
225
                    argv=args,
226
                    input_digest=digest,
227
                    working_directory=dir_path,
228
                    output_directories=["."],
229
                    env=env,
230
                    description=f"Process `go generate` directives in file: {os.path.join(dir_path, go_file)}",
231
                ),
232
            ),
233
        )
234

235
        digest = await add_prefix(AddPrefix(result.output_digest, dir_path))
×
236

237
    return digest
×
238

239

240
@dataclass(frozen=True)
×
241
class OverwriteMergeDigests:
×
242
    orig_digest: Digest
×
243
    new_digest: Digest
×
244

245

246
@rule
×
247
async def merge_digests_with_overwrite(request: OverwriteMergeDigests) -> Digest:
×
248
    orig_snapshot, new_snapshot, orig_digest_entries, new_digest_entries = await concurrently(
×
249
        digest_to_snapshot(request.orig_digest),
250
        digest_to_snapshot(request.new_digest),
251
        get_digest_entries(request.orig_digest),
252
        get_digest_entries(request.new_digest),
253
    )
254

255
    orig_snapshot_grouped = group_by_dir(orig_snapshot.files)
×
256
    new_snapshot_grouped = group_by_dir(new_snapshot.files)
×
257

258
    diff = SnapshotDiff.from_snapshots(orig_snapshot, new_snapshot)
×
259

260
    output_entries: list[FileEntry] = []
×
261

262
    # Keep unchanged original files and directories in the output.
263
    orig_files_to_keep = set(diff.our_unique_files)
×
264
    for dir_path in diff.our_unique_dirs:
×
265
        for filename in orig_snapshot_grouped[dir_path]:
×
266
            orig_files_to_keep.add(os.path.join(dir_path, filename))
×
267
    for entry in orig_digest_entries:
×
268
        if isinstance(entry, FileEntry) and entry.path in orig_files_to_keep:
×
269
            output_entries.append(entry)
×
270

271
    # Add new files/directories and changed files to the output.
272
    new_files_to_keep = {*diff.their_unique_files, *diff.changed_files}
×
273
    for dir_path in diff.their_unique_dirs:
×
274
        for filename in new_snapshot_grouped[dir_path]:
×
275
            new_files_to_keep.add(os.path.join(dir_path, filename))
×
276
    for entry in new_digest_entries:
×
277
        if isinstance(entry, FileEntry) and entry.path in new_files_to_keep:
×
278
            output_entries.append(entry)
×
279

280
    digest = await create_digest(CreateDigest(output_entries))
×
281
    return digest
×
282

283

284
@rule
×
285
async def run_go_package_generators(
×
286
    request: RunPackageGeneratorsRequest,
287
    goroot: GoRoot,
288
    subsystem: GoGenerateGoalSubsystem.EnvironmentAware,
289
) -> RunPackageGeneratorsResult:
290
    build_opts = await go_extract_build_options_from_target(
×
291
        GoBuildOptionsFromTargetRequest(request.address), **implicitly()
292
    )
293
    fallible_analysis, env = await concurrently(
×
294
        analyze_first_party_package(
295
            FirstPartyPkgAnalysisRequest(
296
                request.address, build_opts=build_opts, extra_build_tags=("generate",)
297
            ),
298
            **implicitly(),
299
        ),
300
        environment_vars_subset(EnvironmentVarsRequest(subsystem.env_vars), **implicitly()),
301
    )
302
    if not fallible_analysis.analysis:
×
303
        raise ValueError(f"Analysis failure for {request.address}: {fallible_analysis.stderr}")
×
304
    analysis = fallible_analysis.analysis
×
305
    dir_path = analysis.dir_path if analysis.dir_path else "."
×
306

307
    fallible_pkg_digest = await setup_first_party_pkg_digest(
×
308
        FirstPartyPkgDigestRequest(request.address, build_opts=build_opts)
309
    )
310
    if fallible_pkg_digest.pkg_digest is None:
×
311
        raise ValueError(
×
312
            f"Unable to obtain digest for {request.address}: {fallible_pkg_digest.stderr}"
313
        )
314
    pkg_digest = fallible_pkg_digest.pkg_digest
×
315

316
    # Scan each Go file in the package for generate directives. Process them sequentially so that an error in
317
    # an earlier-processed file prevents later files from being processed.
318
    output_digest = EMPTY_DIGEST
×
319
    for go_file in analysis.go_files:
×
320
        output_digest_for_go_file = await _run_generators(
×
321
            analysis, pkg_digest.digest, dir_path, go_file, goroot, env
322
        )
323
        output_digest = await merge_digests_with_overwrite(
×
324
            OverwriteMergeDigests(output_digest, output_digest_for_go_file)
325
        )
326

327
    return RunPackageGeneratorsResult(output_digest)
×
328

329

330
@goal_rule
×
331
async def go_generate(targets: Targets, workspace: Workspace) -> GoGenerateGoal:
×
332
    go_package_targets = [tgt for tgt in targets if tgt.has_field(GoPackageSourcesField)]
×
333
    results = await concurrently(
×
334
        run_go_package_generators(RunPackageGeneratorsRequest(tgt.address), **implicitly())
335
        for tgt in go_package_targets
336
    )
337
    output_digest = await merge_digests(MergeDigests([r.digest for r in results]))
×
338
    workspace.write_digest(output_digest)
×
339
    return GoGenerateGoal(exit_code=0)
×
340

341

342
def rules():
×
343
    return (
×
344
        *collect_rules(),
345
        *first_party_pkg.rules(),
346
        *goroot.rules(),
347
        *sdk.rules(),
348
    )
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc