26260209689

Committed 21 May 2026 11:59PM UTC coverage: 75.453% (-15.7%) from 91.156%

Build # 26260209689

Build Type

Pull #23365

github

Committed by

web-flow

Commit Message

Merge 5fe873b58 into 7ea655ba0

Pull Request Pull Request #23365: uv.lock -> pex optimization

Coverage Stats

5 of 16 new or added lines in 1 file covered. (31.25%)

10118 existing lines in 378 files now uncovered.

54669 of 72454 relevant lines covered (75.45%)

2.31 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/python/pants/backend/go/goals/generate.py

1	# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
UNCOV 4	from __future__ import annotations	×
5
UNCOV 6	import os	×
UNCOV 7	import re	×
UNCOV 8	import shlex	×
UNCOV 9	import string	×
UNCOV 10	from collections.abc import Mapping	×
UNCOV 11	from dataclasses import dataclass	×
12
UNCOV 13	from pants.backend.go.target_types import GoPackageSourcesField	×
UNCOV 14	from pants.backend.go.util_rules import first_party_pkg, goroot, sdk	×
UNCOV 15	from pants.backend.go.util_rules.build_opts import (	×
16	GoBuildOptionsFromTargetRequest,
17	go_extract_build_options_from_target,
18	)
UNCOV 19	from pants.backend.go.util_rules.first_party_pkg import (	×
20	FirstPartyPkgAnalysis,
21	FirstPartyPkgAnalysisRequest,
22	FirstPartyPkgDigestRequest,
23	analyze_first_party_package,
24	setup_first_party_pkg_digest,
25	)
UNCOV 26	from pants.backend.go.util_rules.goroot import GoRoot	×
UNCOV 27	from pants.build_graph.address import Address	×
UNCOV 28	from pants.core.util_rules.env_vars import environment_vars_subset	×
UNCOV 29	from pants.engine.env_vars import EnvironmentVarsRequest	×
UNCOV 30	from pants.engine.fs import CreateDigest, FileEntry, SnapshotDiff, Workspace	×
UNCOV 31	from pants.engine.goal import Goal, GoalSubsystem	×
UNCOV 32	from pants.engine.internals.native_engine import EMPTY_DIGEST, AddPrefix, Digest, MergeDigests	×
UNCOV 33	from pants.engine.internals.selectors import concurrently	×
UNCOV 34	from pants.engine.intrinsics import (	×
35	add_prefix,
36	create_digest,
37	digest_to_snapshot,
38	get_digest_contents,
39	get_digest_entries,
40	merge_digests,
41	)
UNCOV 42	from pants.engine.process import Process, execute_process_or_raise	×
UNCOV 43	from pants.engine.rules import collect_rules, goal_rule, implicitly, rule	×
UNCOV 44	from pants.engine.target import Targets	×
UNCOV 45	from pants.option.option_types import StrListOption	×
UNCOV 46	from pants.option.subsystem import Subsystem	×
UNCOV 47	from pants.util.dirutil import group_by_dir	×
UNCOV 48	from pants.util.strutil import help_text, softwrap	×
49
50	# Adapted from Go toolchain.
51	# See https://github.com/golang/go/blob/master/src/cmd/go/internal/generate/generate.go and
52	# https://github.com/golang/go/blob/cc1b20e8adf83865a1dbffa259c7a04ef0699b43/src/os/env.go#L16-L96
53	#
54	# Original copyright:
55	# // Copyright 2011 The Go Authors. All rights reserved.
56	# // Use of this source code is governed by a BSD-style
57	# // license that can be found in the LICENSE file.
58
59
UNCOV 60	_GENERATE_DIRECTIVE_RE = re.compile(rb"^//go:generate[ \t](.*)$")	×
61
62
UNCOV 63	class GoGenerateGoalSubsystem(GoalSubsystem):	×
UNCOV 64	name = "go-generate"	×
UNCOV 65	help = help_text(	×
66	"""
67	Run each command in a package described by a `//go:generate` directive. This is equivalent to running
68	`go generate` on a Go package.
69
70	Note: Just like with `go generate`, the `go-generate` goal is never run as part of the build and
71	must be run manually to invoke the commands described by the `//go:generate` directives.
72
73	See https://go.dev/blog/generate for details.
74	"""
75	)
76
UNCOV 77	class EnvironmentAware(Subsystem.EnvironmentAware):	×
UNCOV 78	env_vars = StrListOption(	×
79	default=["LANG", "LC_CTYPE", "LC_ALL", "PATH"],
80	help=softwrap(
81	"""
82	Environment variables to set when invoking generator programs.
83	Entries are either strings in the form `ENV_VAR=value` to set an explicit value;
84	or just `ENV_VAR` to copy the value from Pants's own environment.
85	"""
86	),
87	advanced=True,
88	)
89
90
UNCOV 91	class GoGenerateGoal(Goal):	×
UNCOV 92	subsystem_cls = GoGenerateGoalSubsystem	×
UNCOV 93	environment_behavior = Goal.EnvironmentBehavior.LOCAL_ONLY # TODO(#17129) — Migrate this.	×
94
95
UNCOV 96	@dataclass(frozen=True)	×
UNCOV 97	class RunPackageGeneratorsRequest:	×
UNCOV 98	address: Address	×
UNCOV 99	regex: str \| None = None	×
100
101
UNCOV 102	@dataclass(frozen=True)	×
UNCOV 103	class RunPackageGeneratorsResult:	×
UNCOV 104	digest: Digest	×
105
106
UNCOV 107	_SHELL_SPECIAL_VAR = frozenset(	×
108	["*", "#", "$", "@", "!", "?", "-", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
109	)
UNCOV 110	_ALPHANUMERIC = frozenset([string.ascii_letters, string.digits, "_"])	×
111
112
UNCOV 113	def _get_shell_name(s: str) -> tuple[str, int]:	×
UNCOV 114	if s[0] == "{":	×
UNCOV 115	if len(s) > 2 and s[1] in _SHELL_SPECIAL_VAR and s[2] == "}":	×
UNCOV 116	return s[1:2], 3	×
UNCOV 117	for i in range(1, len(s)):	×
UNCOV 118	if s[i] == "}":	×
UNCOV 119	if i == 1:	×
UNCOV 120	return "", 2 # Bad syntax; eat "${}"	×
UNCOV 121	return s[1:i], i + 1	×
UNCOV 122	return "", 1 # Bad syntax; eat "${"	×
UNCOV 123	elif s[0] in _SHELL_SPECIAL_VAR:	×
UNCOV 124	return s[0:1], 1	×
125
UNCOV 126	i = 0	×
UNCOV 127	while i < len(s) and s[i] in _ALPHANUMERIC:	×
UNCOV 128	i += 1	×
129
UNCOV 130	return s[:i], i	×
131
132
UNCOV 133	def _expand_env(s: str, m: Mapping[str, str]) -> str:	×
UNCOV 134	i = 0	×
UNCOV 135	buf: str \| None = None	×
UNCOV 136	j = 0	×
UNCOV 137	while j < len(s):	×
UNCOV 138	if s[j] == "$" and j + 1 < len(s):	×
UNCOV 139	if buf is None:	×
UNCOV 140	buf = ""	×
UNCOV 141	buf += s[i:j]	×
UNCOV 142	name, w = _get_shell_name(s[j + 1 :])	×
UNCOV 143	if name == "" and w > 0:	×
144	# Encountered invalid syntax; eat the characters.
UNCOV 145	pass	×
UNCOV 146	elif name == "":	×
147	# Valid syntax, but $ was not followed by a name. Leave the dollar character untouched.
UNCOV 148	buf += s[j]	×
149	else:
UNCOV 150	buf += m.get(name, "")	×
UNCOV 151	j += w	×
UNCOV 152	i = j + 1	×
UNCOV 153	j += 1	×
154
UNCOV 155	if buf is None:	×
UNCOV 156	return s	×
UNCOV 157	return buf + s[i:]	×
158
159
UNCOV 160	async def _run_generators(	×
161	analysis: FirstPartyPkgAnalysis,
162	digest: Digest,
163	dir_path: str,
164	go_file: str,
165	goroot: GoRoot,
166	base_env: Mapping[str, str],
167	) -> Digest:
UNCOV 168	digest_contents = await get_digest_contents(digest)	×
UNCOV 169	content: bytes \| None = None	×
UNCOV 170	for entry in digest_contents:	×
UNCOV 171	if entry.path == os.path.join(dir_path, go_file):	×
UNCOV 172	content = entry.content	×
UNCOV 173	break	×
174
UNCOV 175	if content is None:	×
176	raise ValueError("Illegal state: Unable to extract Go file from digest.")	×
177
UNCOV 178	cmd_shorthand: dict[str, tuple[str, ...]] = {}	×
179
UNCOV 180	for line_num, line in enumerate(content.splitlines(), start=1):	×
UNCOV 181	m = _GENERATE_DIRECTIVE_RE.fullmatch(line)	×
UNCOV 182	if not m:	×
UNCOV 183	continue	×
184
185	# Extract the command to run.
186	# Note: Go only processes double-quoted strings. Thus, using shlex.split is actually more liberal than
187	# Go because it also allows single-quoted strings.
UNCOV 188	args = shlex.split(m.group(1).decode())	×
189
190	# Store any command shorthands for later use.
UNCOV 191	if args[0] == "-command":	×
UNCOV 192	if len(args) <= 1:	×
193	raise ValueError(	×
194	f"{go_file}:{line_num}: -command syntax used but no command name specified"
195	)
UNCOV 196	cmd_shorthand[args[1]] = tuple(args[2:])	×
UNCOV 197	continue	×
198
199	# Replace any shorthand command with the previously-stored arguments.
UNCOV 200	if args[0] in cmd_shorthand:	×
UNCOV 201	args = [cmd_shorthand[args[0]], args[1:]]	×
202
203	# If the program calls for `go`, then use the full path to the `go` binary in the GOROOT.
UNCOV 204	if args[0] == "go":	×
205	args[0] = os.path.join(goroot.path, "bin", "go")	×
206
UNCOV 207	env = {	×
208	"GOOS": goroot.goos,
209	"GOARCH": goroot.goarch,
210	"GOFILE": go_file,
211	"GOLINE": str(line_num),
212	"GOPACKAGE": analysis.name,
213	"GOROOT": goroot.path,
214	"DOLLAR": "$",
215	**base_env,
216	}
217
UNCOV 218	for i, arg in enumerate(args):	×
UNCOV 219	args[i] = _expand_env(arg, env)	×
220
221	# Invoke the subprocess and store its output for use as input root of next command (if any).
UNCOV 222	result = await execute_process_or_raise(	×
223	**implicitly(
224	Process(
225	argv=args,
226	input_digest=digest,
227	working_directory=dir_path,
228	output_directories=["."],
229	env=env,
230	description=f"Process `go generate` directives in file: {os.path.join(dir_path, go_file)}",
231	),
232	),
233	)
234
UNCOV 235	digest = await add_prefix(AddPrefix(result.output_digest, dir_path))	×
236
UNCOV 237	return digest	×
238
239
UNCOV 240	@dataclass(frozen=True)	×
UNCOV 241	class OverwriteMergeDigests:	×
UNCOV 242	orig_digest: Digest	×
UNCOV 243	new_digest: Digest	×
244
245
UNCOV 246	@rule	×
UNCOV 247	async def merge_digests_with_overwrite(request: OverwriteMergeDigests) -> Digest:	×
UNCOV 248	orig_snapshot, new_snapshot, orig_digest_entries, new_digest_entries = await concurrently(	×
249	digest_to_snapshot(request.orig_digest),
250	digest_to_snapshot(request.new_digest),
251	get_digest_entries(request.orig_digest),
252	get_digest_entries(request.new_digest),
253	)
254
UNCOV 255	orig_snapshot_grouped = group_by_dir(orig_snapshot.files)	×
UNCOV 256	new_snapshot_grouped = group_by_dir(new_snapshot.files)	×
257
UNCOV 258	diff = SnapshotDiff.from_snapshots(orig_snapshot, new_snapshot)	×
259
UNCOV 260	output_entries: list[FileEntry] = []	×
261
262	# Keep unchanged original files and directories in the output.
UNCOV 263	orig_files_to_keep = set(diff.our_unique_files)	×
UNCOV 264	for dir_path in diff.our_unique_dirs:	×
UNCOV 265	for filename in orig_snapshot_grouped[dir_path]:	×
UNCOV 266	orig_files_to_keep.add(os.path.join(dir_path, filename))	×
UNCOV 267	for entry in orig_digest_entries:	×
UNCOV 268	if isinstance(entry, FileEntry) and entry.path in orig_files_to_keep:	×
UNCOV 269	output_entries.append(entry)	×
270
271	# Add new files/directories and changed files to the output.
UNCOV 272	new_files_to_keep = {diff.their_unique_files, diff.changed_files}	×
UNCOV 273	for dir_path in diff.their_unique_dirs:	×
UNCOV 274	for filename in new_snapshot_grouped[dir_path]:	×
UNCOV 275	new_files_to_keep.add(os.path.join(dir_path, filename))	×
UNCOV 276	for entry in new_digest_entries:	×
UNCOV 277	if isinstance(entry, FileEntry) and entry.path in new_files_to_keep:	×
UNCOV 278	output_entries.append(entry)	×
279
UNCOV 280	digest = await create_digest(CreateDigest(output_entries))	×
UNCOV 281	return digest	×
282
283
UNCOV 284	@rule	×
UNCOV 285	async def run_go_package_generators(	×
286	request: RunPackageGeneratorsRequest,
287	goroot: GoRoot,
288	subsystem: GoGenerateGoalSubsystem.EnvironmentAware,
289	) -> RunPackageGeneratorsResult:
UNCOV 290	build_opts = await go_extract_build_options_from_target(	×
291	GoBuildOptionsFromTargetRequest(request.address), **implicitly()
292	)
UNCOV 293	fallible_analysis, env = await concurrently(	×
294	analyze_first_party_package(
295	FirstPartyPkgAnalysisRequest(
296	request.address, build_opts=build_opts, extra_build_tags=("generate",)
297	),
298	**implicitly(),
299	),
300	environment_vars_subset(EnvironmentVarsRequest(subsystem.env_vars), **implicitly()),
301	)
UNCOV 302	if not fallible_analysis.analysis:	×
303	raise ValueError(f"Analysis failure for {request.address}: {fallible_analysis.stderr}")	×
UNCOV 304	analysis = fallible_analysis.analysis	×
UNCOV 305	dir_path = analysis.dir_path if analysis.dir_path else "."	×
306
UNCOV 307	fallible_pkg_digest = await setup_first_party_pkg_digest(	×
308	FirstPartyPkgDigestRequest(request.address, build_opts=build_opts)
309	)
UNCOV 310	if fallible_pkg_digest.pkg_digest is None:	×
311	raise ValueError(	×
312	f"Unable to obtain digest for {request.address}: {fallible_pkg_digest.stderr}"
313	)
UNCOV 314	pkg_digest = fallible_pkg_digest.pkg_digest	×
315
316	# Scan each Go file in the package for generate directives. Process them sequentially so that an error in
317	# an earlier-processed file prevents later files from being processed.
UNCOV 318	output_digest = EMPTY_DIGEST	×
UNCOV 319	for go_file in analysis.go_files:	×
UNCOV 320	output_digest_for_go_file = await _run_generators(	×
321	analysis, pkg_digest.digest, dir_path, go_file, goroot, env
322	)
UNCOV 323	output_digest = await merge_digests_with_overwrite(	×
324	OverwriteMergeDigests(output_digest, output_digest_for_go_file)
325	)
326
UNCOV 327	return RunPackageGeneratorsResult(output_digest)	×
328
329
UNCOV 330	@goal_rule	×
UNCOV 331	async def go_generate(targets: Targets, workspace: Workspace) -> GoGenerateGoal:	×
UNCOV 332	go_package_targets = [tgt for tgt in targets if tgt.has_field(GoPackageSourcesField)]	×
UNCOV 333	results = await concurrently(	×
334	run_go_package_generators(RunPackageGeneratorsRequest(tgt.address), **implicitly())
335	for tgt in go_package_targets
336	)
UNCOV 337	output_digest = await merge_digests(MergeDigests([r.digest for r in results]))	×
UNCOV 338	workspace.write_digest(output_digest)	×
UNCOV 339	return GoGenerateGoal(exit_code=0)	×
340
341
UNCOV 342	def rules():	×
UNCOV 343	return (	×
344	*collect_rules(),
345	*first_party_pkg.rules(),
346	*goroot.rules(),
347	*sdk.rules(),
348	)

pantsbuild / pants / 26260209689

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous