19015773527

Committed 02 Nov 2025 05:33PM UTC coverage: 17.872% (-62.4%) from 80.3%

Build # 19015773527

Build Type

Pull #22816

github

Committed by

web-flow

Commit Message

Merge a12d75757 into 6c024e162

Pull Request Pull Request #22816: Update Pants internal Python to 3.14

Run Details

4 of 5 new or added lines in 3 files covered. (80.0%)

28452 existing lines in 683 files now uncovered.

9831 of 55007 relevant lines covered (17.87%)

0.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/python/pants/backend/python/goals/lockfile.py

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
UNCOV 4	from __future__ import annotations	×
5
UNCOV 6	import itertools	×
UNCOV 7	import os.path	×
UNCOV 8	from collections import defaultdict	×
UNCOV 9	from dataclasses import dataclass	×
UNCOV 10	from operator import itemgetter	×
11
UNCOV 12	from pants.backend.python.subsystems.python_tool_base import PythonToolBase	×
UNCOV 13	from pants.backend.python.subsystems.setup import PythonSetup	×
UNCOV 14	from pants.backend.python.target_types import (	×
15	PythonRequirementFindLinksField,
16	PythonRequirementResolveField,
17	PythonRequirementsField,
18	)
UNCOV 19	from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints	×
UNCOV 20	from pants.backend.python.util_rules.lockfile_diff import _generate_python_lockfile_diff	×
UNCOV 21	from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadata	×
UNCOV 22	from pants.backend.python.util_rules.pex import find_interpreter	×
UNCOV 23	from pants.backend.python.util_rules.pex_cli import PexCliProcess, maybe_log_pex_stderr	×
UNCOV 24	from pants.backend.python.util_rules.pex_environment import PexSubsystem	×
UNCOV 25	from pants.backend.python.util_rules.pex_requirements import (	×
26	PexRequirements,
27	ResolvePexConfig,
28	ResolvePexConfigRequest,
29	determine_resolve_pex_config,
30	)
UNCOV 31	from pants.core.goals.generate_lockfiles import (	×
32	DEFAULT_TOOL_LOCKFILE,
33	GenerateLockfile,
34	GenerateLockfileResult,
35	GenerateLockfilesSubsystem,
36	KnownUserResolveNames,
37	KnownUserResolveNamesRequest,
38	RequestedUserResolveNames,
39	UserGenerateLockfiles,
40	WrappedGenerateLockfile,
41	)
UNCOV 42	from pants.core.goals.resolves import ExportableTool	×
UNCOV 43	from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest	×
UNCOV 44	from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests	×
UNCOV 45	from pants.engine.internals.synthetic_targets import SyntheticAddressMaps, SyntheticTargetsRequest	×
UNCOV 46	from pants.engine.internals.target_adaptor import TargetAdaptor	×
UNCOV 47	from pants.engine.intrinsics import create_digest, get_digest_contents, merge_digests	×
UNCOV 48	from pants.engine.process import ProcessCacheScope, fallible_to_exec_result_or_raise	×
UNCOV 49	from pants.engine.rules import collect_rules, implicitly, rule	×
UNCOV 50	from pants.engine.target import AllTargets	×
UNCOV 51	from pants.engine.unions import UnionMembership, UnionRule	×
UNCOV 52	from pants.option.subsystem import _construct_subsystem	×
UNCOV 53	from pants.util.docutil import bin_name	×
UNCOV 54	from pants.util.logging import LogLevel	×
UNCOV 55	from pants.util.ordered_set import FrozenOrderedSet	×
UNCOV 56	from pants.util.pip_requirement import PipRequirement	×
57
58
UNCOV 59	@dataclass(frozen=True)	×
UNCOV 60	class GeneratePythonLockfile(GenerateLockfile):	×
UNCOV 61	requirements: FrozenOrderedSet[str]	×
UNCOV 62	find_links: FrozenOrderedSet[str]	×
UNCOV 63	interpreter_constraints: InterpreterConstraints	×
64
UNCOV 65	@property	×
UNCOV 66	def requirements_hex_digest(self) -> str:	×
67	"""Produces a hex digest of the requirements input for this lockfile."""
68	return calculate_invalidation_digest(self.requirements)	×
69
70
UNCOV 71	@rule	×
UNCOV 72	async def wrap_python_lockfile_request(request: GeneratePythonLockfile) -> WrappedGenerateLockfile:	×
73	return WrappedGenerateLockfile(request)	×
74
75
UNCOV 76	@dataclass(frozen=True)	×
UNCOV 77	class _PipArgsAndConstraintsSetup:	×
UNCOV 78	resolve_config: ResolvePexConfig	×
UNCOV 79	args: tuple[str, ...]	×
UNCOV 80	digest: Digest	×
81
82
UNCOV 83	async def _setup_pip_args_and_constraints_file(resolve_name: str) -> _PipArgsAndConstraintsSetup:	×
84	resolve_config = await determine_resolve_pex_config(	×
85	ResolvePexConfigRequest(resolve_name), **implicitly()
86	)
87
88	args = list(resolve_config.pex_args())	×
89	digests: list[Digest] = []	×
90
91	if resolve_config.constraints_file:	×
92	args.append(f"--constraints={resolve_config.constraints_file.path}")	×
93	digests.append(resolve_config.constraints_file.digest)	×
94
95	input_digest = await merge_digests(MergeDigests(digests))	×
96	return _PipArgsAndConstraintsSetup(resolve_config, tuple(args), input_digest)	×
97
98
UNCOV 99	@rule(desc="Generate Python lockfile", level=LogLevel.DEBUG)	×
UNCOV 100	async def generate_lockfile(	×
101	req: GeneratePythonLockfile,
102	generate_lockfiles_subsystem: GenerateLockfilesSubsystem,
103	python_setup: PythonSetup,
104	pex_subsystem: PexSubsystem,
105	) -> GenerateLockfileResult:
106	if not req.requirements:	×
107	raise ValueError(	×
108	f"Cannot generate lockfile with no requirements. Please add some requirements to {req.resolve_name}."
109	)
110
111	pip_args_setup = await _setup_pip_args_and_constraints_file(req.resolve_name)	×
112	header_delimiter = "//"	×
113
114	python = await find_interpreter(req.interpreter_constraints, **implicitly())	×
115
116	result = await fallible_to_exec_result_or_raise(	×
117	**implicitly(
118	PexCliProcess(
119	subcommand=("lock", "create"),
120	extra_args=(
121	f"--output={req.lockfile_dest}",
122	# See https://github.com/pantsbuild/pants/issues/12458. For now, we always
123	# generate universal locks because they have the best compatibility. We may
124	# want to let users change this, as `style=strict` is safer.
125	"--style=universal",
126	"--pip-version",
127	python_setup.pip_version,
128	"--resolver-version",
129	"pip-2020-resolver",
130	"--preserve-pip-download-log",
131	"pex-pip-download.log",
132	# PEX files currently only run on Linux and Mac machines; so we hard code this
133	# limit on lock universality to avoid issues locking due to irrelevant
134	# Windows-only dependency issues. See this Pex issue that originated from a
135	# Pants user issue presented in Slack:
136	# https://github.com/pex-tool/pex/issues/1821
137	#
138	# At some point it will probably make sense to expose `--target-system` for
139	# configuration.
140	"--target-system",
141	"linux",
142	"--target-system",
143	"mac",
144	# This makes diffs more readable when lockfiles change.
145	"--indent=2",
146	f"--python-path={python.path}",
147	*(f"--find-links={link}" for link in req.find_links),
148	*pip_args_setup.args,
149	*req.interpreter_constraints.generate_pex_arg_list(),
150	*(
151	f"--override={override}"
152	for override in pip_args_setup.resolve_config.overrides
153	),
154	*req.requirements,
155	),
156	additional_input_digest=pip_args_setup.digest,
157	output_files=(req.lockfile_dest,),
158	description=f"Generate lockfile for {req.resolve_name}",
159	# Instead of caching lockfile generation with LMDB, we instead use the invalidation
160	# scheme from `lockfile_metadata.py` to check for stale/invalid lockfiles. This is
161	# necessary so that our invalidation is resilient to deleting LMDB or running on a
162	# new machine.
163	#
164	# We disable caching with LMDB so that when you generate a lockfile, you always get
165	# the most up-to-date snapshot of the world. This is generally desirable and also
166	# necessary to avoid an awkward edge case where different developers generate
167	# different lockfiles even when generating at the same time. See
168	# https://github.com/pantsbuild/pants/issues/12591.
169	cache_scope=ProcessCacheScope.PER_SESSION,
170	)
171	)
172	)
173	maybe_log_pex_stderr(result.stderr, pex_subsystem.verbosity)	×
174
175	metadata = PythonLockfileMetadata.new(	×
176	valid_for_interpreter_constraints=req.interpreter_constraints,
177	requirements={
178	PipRequirement.parse(
179	i,
180	description_of_origin=f"the lockfile {req.lockfile_dest} for the resolve {req.resolve_name}",
181	)
182	for i in req.requirements
183	},
184	manylinux=pip_args_setup.resolve_config.manylinux,
185	requirement_constraints=(
186	set(pip_args_setup.resolve_config.constraints_file.constraints)
187	if pip_args_setup.resolve_config.constraints_file
188	else set()
189	),
190	only_binary=set(pip_args_setup.resolve_config.only_binary),
191	no_binary=set(pip_args_setup.resolve_config.no_binary),
192	excludes=set(pip_args_setup.resolve_config.excludes),
193	overrides=set(pip_args_setup.resolve_config.overrides),
194	sources=set(pip_args_setup.resolve_config.sources),
195	)
196	regenerate_command = (	×
197	generate_lockfiles_subsystem.custom_command
198	or f"{bin_name()} generate-lockfiles --resolve={req.resolve_name}"
199	)
200	if python_setup.separate_lockfile_metadata_file:	×
201	descr = f"This lockfile was generated by Pants. To regenerate, run: {regenerate_command}"	×
202	metadata_digest = await create_digest(	×
203	CreateDigest(
204	[
205	FileContent(
206	PythonLockfileMetadata.metadata_location_for_lockfile(req.lockfile_dest),
207	metadata.to_json(with_description=descr).encode(),
208	),
209	]
210	)
211	)
212	final_lockfile_digest = await merge_digests(	×
213	MergeDigests([metadata_digest, result.output_digest])
214	)
215	else:
216	initial_lockfile_digest_contents = await get_digest_contents(result.output_digest)	×
217	lockfile_with_header = metadata.add_header_to_lockfile(	×
218	initial_lockfile_digest_contents[0].content,
219	regenerate_command=regenerate_command,
220	delimeter=header_delimiter,
221	)
222	final_lockfile_digest = await create_digest(	×
223	CreateDigest(
224	[
225	FileContent(req.lockfile_dest, lockfile_with_header),
226	]
227	)
228	)
229
230	if req.diff:	×
231	diff = await _generate_python_lockfile_diff(	×
232	final_lockfile_digest, req.resolve_name, req.lockfile_dest
233	)
234	else:
235	diff = None	×
236
237	return GenerateLockfileResult(final_lockfile_digest, req.resolve_name, req.lockfile_dest, diff)	×
238
239
UNCOV 240	class RequestedPythonUserResolveNames(RequestedUserResolveNames):	×
UNCOV 241	pass	×
242
243
UNCOV 244	class KnownPythonUserResolveNamesRequest(KnownUserResolveNamesRequest):	×
UNCOV 245	pass	×
246
247
UNCOV 248	@rule	×
UNCOV 249	async def determine_python_user_resolves(	×
250	_: KnownPythonUserResolveNamesRequest,
251	python_setup: PythonSetup,
252	union_membership: UnionMembership,
253	) -> KnownUserResolveNames:
254	"""Find all know Python resolves, from both user-created resolves and internal tools."""
255	python_tool_resolves = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase)	×
256
257	tools_using_default_resolve = [	×
258	resolve_name
259	for resolve_name, subsystem_cls in python_tool_resolves.items()
260	if (await _construct_subsystem(subsystem_cls)).install_from_resolve is None
261	]
262
263	return KnownUserResolveNames(	×
264	names=(
265	*python_setup.resolves.keys(),
266	*tools_using_default_resolve,
267	), # the order of the keys doesn't matter since shadowing is done in `setup_user_lockfile_requests`
268	option_name="[python].resolves",
269	requested_resolve_names_cls=RequestedPythonUserResolveNames,
270	)
271
272
UNCOV 273	@rule	×
UNCOV 274	async def setup_user_lockfile_requests(	×
275	requested: RequestedPythonUserResolveNames,
276	all_targets: AllTargets,
277	python_setup: PythonSetup,
278	union_membership: UnionMembership,
279	) -> UserGenerateLockfiles:
280	"""Transform the names of resolves requested into the `GeneratePythonLockfile` request object.
281
282	Shadowing is done here by only checking internal resolves if the resolve is not a user-created
283	resolve.
284	"""
285	if not (python_setup.enable_resolves and python_setup.resolves_generate_lockfiles):	×
286	return UserGenerateLockfiles()	×
287
288	resolve_to_requirements_fields = defaultdict(set)	×
289	find_links: set[str] = set()	×
290	for tgt in all_targets:	×
291	if not tgt.has_fields((PythonRequirementResolveField, PythonRequirementsField)):	×
292	continue	×
293	resolve = tgt[PythonRequirementResolveField].normalized_value(python_setup)	×
294	resolve_to_requirements_fields[resolve].add(tgt[PythonRequirementsField])	×
295	find_links.update(tgt[PythonRequirementFindLinksField].value or ())	×
296
297	tools = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase)	×
298
299	out = set()	×
300	for resolve in requested:	×
301	if resolve in python_setup.resolves:	×
302	out.add(	×
303	GeneratePythonLockfile(
304	requirements=PexRequirements.req_strings_from_requirement_fields(
305	resolve_to_requirements_fields[resolve]
306	),
307	find_links=FrozenOrderedSet(find_links),
308	interpreter_constraints=InterpreterConstraints(
309	python_setup.resolves_to_interpreter_constraints.get(
310	resolve, python_setup.interpreter_constraints
311	)
312	),
313	resolve_name=resolve,
314	lockfile_dest=python_setup.resolves[resolve],
315	diff=False,
316	)
317	)
318	else:
319	tool_cls: type[PythonToolBase] = tools[resolve]	×
320	tool = await _construct_subsystem(tool_cls)	×
321
322	# TODO: we shouldn't be managing default ICs in lockfile identification.
323	# We should find a better place to do this or a better way to default
324	if tool.register_interpreter_constraints:	×
325	ic = tool.interpreter_constraints	×
326	else:
327	ic = InterpreterConstraints(tool.default_interpreter_constraints)	×
328
329	out.add(	×
330	GeneratePythonLockfile(
331	requirements=FrozenOrderedSet(sorted(tool.requirements)),
332	find_links=FrozenOrderedSet(find_links),
333	interpreter_constraints=ic,
334	resolve_name=resolve,
335	lockfile_dest=DEFAULT_TOOL_LOCKFILE,
336	diff=False,
337	)
338	)
339
340	return UserGenerateLockfiles(out)	×
341
342
UNCOV 343	@dataclass(frozen=True)	×
UNCOV 344	class PythonSyntheticLockfileTargetsRequest(SyntheticTargetsRequest):	×
345	"""Register the type used to create synthetic targets for Python lockfiles.
346
347	As the paths for all lockfiles are known up-front, we set the `path` field to
348	`SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS` so that we get a single request for all
349	our synthetic targets rather than one request per directory.
350	"""
351
UNCOV 352	path: str = SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS	×
353
354
UNCOV 355	def synthetic_lockfile_target_name(resolve: str) -> str:	×
356	return f"_{resolve}_lockfile"	×
357
358
UNCOV 359	@rule	×
UNCOV 360	async def python_lockfile_synthetic_targets(	×
361	request: PythonSyntheticLockfileTargetsRequest,
362	python_setup: PythonSetup,
363	) -> SyntheticAddressMaps:
364	if not python_setup.enable_synthetic_lockfiles:	×
365	return SyntheticAddressMaps()	×
366
367	resolves = [	×
368	(os.path.dirname(lockfile), os.path.basename(lockfile), name)
369	for name, lockfile in python_setup.resolves.items()
370	]
371
372	return SyntheticAddressMaps.for_targets_request(	×
373	request,
374	[
375	(
376	os.path.join(spec_path, "BUILD.python-lockfiles"),
377	tuple(
378	TargetAdaptor(
379	"_lockfiles",
380	name=synthetic_lockfile_target_name(name),
381	sources=(lockfile,),
382	__description_of_origin__=f"the [python].resolves option {name!r}",
383	)
384	for _, lockfile, name in lockfiles
385	),
386	)
387	for spec_path, lockfiles in itertools.groupby(sorted(resolves), key=itemgetter(0))
388	],
389	)
390
391
UNCOV 392	def rules():	×
UNCOV 393	return (	×
394	*collect_rules(),
395	UnionRule(GenerateLockfile, GeneratePythonLockfile),
396	UnionRule(KnownUserResolveNamesRequest, KnownPythonUserResolveNamesRequest),
397	UnionRule(RequestedUserResolveNames, RequestedPythonUserResolveNames),
398	UnionRule(SyntheticTargetsRequest, PythonSyntheticLockfileTargetsRequest),
399	)

pantsbuild / pants / 19015773527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous