19015773527

Committed 02 Nov 2025 05:33PM UTC coverage: 17.872% (-62.4%) from 80.3%

Build # 19015773527

Build Type

Pull #22816

github

Committed by

web-flow

Commit Message

Merge a12d75757 into 6c024e162

Pull Request Pull Request #22816: Update Pants internal Python to 3.14

Run Details

4 of 5 new or added lines in 3 files covered. (80.0%)

28452 existing lines in 683 files now uncovered.

9831 of 55007 relevant lines covered (17.87%)

0.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/python/pants/backend/python/util_rules/local_dists_pep660.py

1	# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
UNCOV 4	from __future__ import annotations	×
5
UNCOV 6	import json	×
UNCOV 7	import logging	×
UNCOV 8	import os	×
UNCOV 9	import shlex	×
UNCOV 10	from collections import defaultdict	×
UNCOV 11	from dataclasses import dataclass	×
12
UNCOV 13	from pants.backend.python.subsystems.setup import PythonSetup	×
UNCOV 14	from pants.backend.python.target_types import PythonProvidesField, PythonResolveField	×
UNCOV 15	from pants.backend.python.util_rules import package_dists	×
UNCOV 16	from pants.backend.python.util_rules.dists import BuildBackendError, DistBuildRequest	×
UNCOV 17	from pants.backend.python.util_rules.dists import rules as dists_rules	×
UNCOV 18	from pants.backend.python.util_rules.package_dists import (	×
19	DependencyOwner,
20	ExportedTarget,
21	create_dist_build_request,
22	get_owned_dependencies,
23	)
UNCOV 24	from pants.backend.python.util_rules.pex import PexRequest, VenvPexProcess, create_venv_pex	×
UNCOV 25	from pants.base.build_root import BuildRoot	×
UNCOV 26	from pants.core.util_rules import system_binaries	×
UNCOV 27	from pants.core.util_rules.system_binaries import BashBinary, UnzipBinary	×
UNCOV 28	from pants.engine.fs import (	×
29	CreateDigest,
30	Digest,
31	DigestSubset,
32	FileContent,
33	MergeDigests,
34	PathGlobs,
35	RemovePrefix,
36	)
UNCOV 37	from pants.engine.internals.graph import resolve_target	×
UNCOV 38	from pants.engine.internals.native_engine import Address	×
UNCOV 39	from pants.engine.intrinsics import create_digest, digest_to_snapshot, merge_digests, remove_prefix	×
UNCOV 40	from pants.engine.process import Process, fallible_to_exec_result_or_raise	×
UNCOV 41	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	×
UNCOV 42	from pants.engine.target import AllTargets, Target, Targets, WrappedTargetRequest	×
UNCOV 43	from pants.engine.unions import UnionMembership	×
UNCOV 44	from pants.util.docutil import doc_url	×
UNCOV 45	from pants.util.frozendict import FrozenDict	×
UNCOV 46	from pants.util.logging import LogLevel	×
UNCOV 47	from pants.util.osutil import is_macos_big_sur	×
UNCOV 48	from pants.util.resources import read_resource	×
UNCOV 49	from pants.util.strutil import softwrap	×
50
UNCOV 51	logger = logging.getLogger(__name__)	×
52
53
UNCOV 54	_scripts_package = "pants.backend.python.util_rules.scripts"	×
55
56
UNCOV 57	@dataclass(frozen=True)	×
UNCOV 58	class PEP660BuildResult:	×
UNCOV 59	output: Digest	×
60	# Relpaths in the output digest.
UNCOV 61	editable_wheel_path: str \| None	×
62
63
UNCOV 64	def dump_backend_wrapper_json(	×
65	dist_dir: str,
66	pth_file_path: str,
67	direct_url: str,
68	request: DistBuildRequest,
69	) -> bytes:
70	"""Build the settings json for our PEP 517 / PEP 660 wrapper script."""
71
72	def clean_config_settings(	×
73	cs: FrozenDict[str, tuple[str, ...]] \| None,
74	) -> dict[str, list[str]] \| None:
75	# setuptools.build_meta expects list values and chokes on tuples.
76	# We assume/hope that other backends accept lists as well.
77	return None if cs is None else {k: list(v) for k, v in cs.items()}	×
78
79	# tag the editable wheel as widely compatible
80	lang_tag, abi_tag, platform_tag = "py3", "none", "any"	×
81	if request.interpreter_constraints.includes_python2():	×
82	# Assume everything has py3 support. If not, we'll need a new includes_python3 method.
83	lang_tag = "py2.py3"	×
84
85	settings = {	×
86	"build_backend": request.build_system.build_backend,
87	"dist_dir": dist_dir,
88	"pth_file_path": pth_file_path,
89	"wheel_config_settings": clean_config_settings(request.wheel_config_settings),
90	"tags": "-".join([lang_tag, abi_tag, platform_tag]),
91	"direct_url": direct_url,
92	}
93	return json.dumps(settings).encode()	×
94
95
UNCOV 96	@rule	×
UNCOV 97	async def run_pep660_build(	×
98	request: DistBuildRequest, python_setup: PythonSetup, build_root: BuildRoot
99	) -> PEP660BuildResult:
100	"""Run our PEP 517 / PEP 660 wrapper script to generate an editable wheel.
101
102	The PEP 517 / PEP 660 wrapper script is responsible for building the editable wheel.
103	The backend wrapper script, along with the commands that install the editable wheel,
104	need to conform to the following specs so that Pants is a PEP 660 compliant frontend,
105	a PEP 660 compliant backend, and that it builds a compliant wheel and install.
106
107	NOTE: PEP 660 does not address the `.data` directory, so the wrapper ignores it.
108
109	Relevant Specs:
110	https://peps.python.org/pep-0517/
111	https://peps.python.org/pep-0660/
112	https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/
113	https://packaging.python.org/en/latest/specifications/recording-installed-packages/
114	https://packaging.python.org/en/latest/specifications/direct-url-data-structure/
115	https://packaging.python.org/en/latest/specifications/binary-distribution-format/
116	"""
117	dist_abs_path = (	×
118	build_root.path
119	if request.dist_source_root == "."
120	else str(build_root.pathlib_path / request.dist_source_root)
121	)
122	direct_url = "file://" + dist_abs_path.replace(os.path.sep, "/")	×
123
124	# Create the .pth files to add the relevant source root to sys.path.
125	# We cannot use the build backend to do this because we do not want to tell
126	# it where the workspace is and risk it adding anything there.
127	# NOTE: We use .pth files to support ICs less than python3.7.
128	# A future enhancement might be to provide more precise editable
129	# wheels based on https://pypi.org/project/editables/, but that only
130	# supports python3.7+ (what pip supports as of April 2023).
131	# Or maybe do something like setuptools strict editable wheel.
132	pth_file_contents = ""	×
133	for source_root in request.build_time_source_roots: # NB: the roots are sorted	×
134	# Can we use just the dist_abs_path instead of including all source roots?
135	abs_path = (	×
136	build_root.path if source_root == "." else str(build_root.pathlib_path / source_root)
137	)
138	pth_file_contents += f"{abs_path}\n"	×
139	pth_file_name = "__pants__.pth"	×
140	pth_file_path = os.path.join(request.working_directory, pth_file_name)	×
141
142	# This is the setuptools dist directory, not Pants's, so we hardcode to dist/.
143	dist_dir = "dist"	×
144	dist_output_dir = os.path.join(dist_dir, request.output_path)	×
145
146	backend_wrapper_json = "backend_wrapper.json"	×
147	backend_wrapper_json_path = os.path.join(request.working_directory, backend_wrapper_json)	×
148	backend_wrapper_name = "backend_wrapper.py"	×
149	backend_wrapper_path = os.path.join(request.working_directory, backend_wrapper_name)	×
150	backend_wrapper_content = read_resource(_scripts_package, "pep660_backend_wrapper.py")	×
151	assert backend_wrapper_content is not None	×
152
153	conf_digest, backend_wrapper_digest, build_backend_pex = await concurrently(	×
154	create_digest(
155	CreateDigest(
156	[
157	FileContent(pth_file_path, pth_file_contents.encode()),
158	FileContent(
159	backend_wrapper_json_path,
160	dump_backend_wrapper_json(
161	dist_output_dir, pth_file_name, direct_url, request
162	),
163	),
164	]
165	)
166	),
167	# The backend_wrapper has its own digest for cache reuse.
168	create_digest(CreateDigest([FileContent(backend_wrapper_path, backend_wrapper_content)])),
169	# Note that this pex has no entrypoint. We use it to run our wrapper, which
170	# in turn imports from and invokes the build backend.
171	create_venv_pex(
172	**implicitly(
173	PexRequest(
174	output_filename="build_backend.pex",
175	internal_only=True,
176	requirements=request.build_system.requires,
177	pex_path=request.extra_build_time_requirements,
178	interpreter_constraints=request.interpreter_constraints,
179	)
180	)
181	),
182	)
183
184	merged_digest = await merge_digests(	×
185	MergeDigests((request.input, conf_digest, backend_wrapper_digest))
186	)
187
188	extra_env = {	×
189	**(request.extra_build_time_env or {}),
190	"PEX_EXTRA_SYS_PATH": os.pathsep.join(request.build_time_source_roots),
191	}
192	if python_setup.macos_big_sur_compatibility and is_macos_big_sur():	×
193	extra_env["MACOSX_DEPLOYMENT_TARGET"] = "10.16"	×
194
195	result = await fallible_to_exec_result_or_raise(	×
196	**implicitly(
197	VenvPexProcess(
198	build_backend_pex,
199	argv=(backend_wrapper_name, backend_wrapper_json),
200	input_digest=merged_digest,
201	extra_env=extra_env,
202	working_directory=request.working_directory,
203	output_directories=(dist_dir,), # Relative to the working_directory.
204	description=(
205	f"Run {request.build_system.build_backend} to gather .dist-info for {request.target_address_spec}"
206	if request.target_address_spec
207	else f"Run {request.build_system.build_backend} to gather .dist-info"
208	),
209	level=LogLevel.DEBUG,
210	)
211	)
212	)
213	output_lines = result.stdout.decode().splitlines()	×
214	line_prefix = "editable_path: "	×
215	editable_path = ""	×
216	for line in output_lines:	×
217	if line.startswith(line_prefix):	×
218	editable_path = os.path.join(request.output_path, line[len(line_prefix) :].strip())	×
219	break	×
220
221	# Note that output_digest paths are relative to the working_directory.
222	output_digest = await remove_prefix(RemovePrefix(result.output_digest, dist_dir))	×
223	output_snapshot = await digest_to_snapshot(output_digest)	×
224	if editable_path not in output_snapshot.files:	×
225	raise BuildBackendError(	×
226	softwrap(
227	f"""
228	Failed to build PEP 660 editable wheel {editable_path}
229	(extracted dist-info from PEP 517 build backend
230	{request.build_system.build_backend}).
231	"""
232	)
233	)
234	return PEP660BuildResult(output_digest, editable_wheel_path=editable_path)	×
235
236
UNCOV 237	@dataclass(frozen=True)	×
UNCOV 238	class LocalDistPEP660Wheels:	×
239	"""Contains the PEP 660 "editable" wheels isolated from a single local Python distribution."""
240
UNCOV 241	pep660_wheel_paths: tuple[str, ...]	×
UNCOV 242	pep660_wheels_digest: Digest	×
UNCOV 243	provided_files: frozenset[str]	×
244
245
UNCOV 246	@rule	×
UNCOV 247	async def isolate_local_dist_pep660_wheels(	×
248	dist_target_address: Address,
249	bash: BashBinary,
250	unzip_binary: UnzipBinary,
251	python_setup: PythonSetup,
252	union_membership: UnionMembership,
253	) -> LocalDistPEP660Wheels:
254	dist_build_request = await create_dist_build_request(	×
255	dist_target_address=dist_target_address,
256	python_setup=python_setup,
257	union_membership=union_membership,
258	# editable wheel ignores build_wheel+build_sdist args
259	validate_wheel_sdist=False,
260	)
261	pep660_result = await run_pep660_build(dist_build_request, **implicitly())	×
262
263	# the output digest should only contain wheels, but filter to be safe.
264	wheels_snapshot = await digest_to_snapshot(	×
265	implicitly(DigestSubset(pep660_result.output, PathGlobs(["/*.whl"])))
266	)
267
268	wheels = tuple(sorted(wheels_snapshot.files))	×
269
270	if not wheels:	×
271	tgt = await resolve_target(	×
272	WrappedTargetRequest(dist_target_address, description_of_origin="<infallible>"),
273	**implicitly(),
274	)
275	logger.warning(	×
276	softwrap(
277	f"""
278	Encountered a dependency on the {tgt.target.alias} target at {dist_target_address},
279	but this target does not produce a Python wheel artifact. Therefore this target's
280	code will be used directly from sources, without a distribution being built,
281	and any native extensions in it will not be built.
282
283	See {doc_url("docs/python/overview/building-distributions")} for details on how to set up a
284	{tgt.target.alias} target to produce a wheel.
285	"""
286	)
287	)
288
289	wheels_listing_result = await fallible_to_exec_result_or_raise(	×
290	**implicitly(
291	Process(
292	argv=[
293	bash.path,
294	"-c",
295	f"""
296	set -ex
297	for f in {" ".join(shlex.quote(f) for f in wheels)}; do
298	{unzip_binary.path} -Z1 "$f"
299	done
300	""",
301	],
302	input_digest=wheels_snapshot.digest,
303	description=f"List contents of editable artifacts produced by {dist_target_address}",
304	)
305	)
306	)
307	provided_files = set(wheels_listing_result.stdout.decode().splitlines())	×
308
309	return LocalDistPEP660Wheels(wheels, wheels_snapshot.digest, frozenset(sorted(provided_files)))	×
310
311
UNCOV 312	@dataclass(frozen=True)	×
UNCOV 313	class AllPythonDistributionTargets:	×
UNCOV 314	targets: Targets	×
315
316
UNCOV 317	@rule(desc="Find all Python Distribution targets in project", level=LogLevel.DEBUG)	×
UNCOV 318	async def find_all_python_distributions(	×
319	all_targets: AllTargets,
320	) -> AllPythonDistributionTargets:
321	return AllPythonDistributionTargets(	×
322	# 'provides' is the field used in PythonDistributionFieldSet
323	Targets(tgt for tgt in all_targets if tgt.has_field(PythonProvidesField))
324	)
325
326
UNCOV 327	@dataclass(frozen=True)	×
UNCOV 328	class ResolveSortedPythonDistributionTargets:	×
UNCOV 329	targets: FrozenDict[str \| None, tuple[Target, ...]]	×
330
331
UNCOV 332	@rule(	×
333	desc="Associate resolves with all Python Distribution targets in project", level=LogLevel.DEBUG
334	)
UNCOV 335	async def sort_all_python_distributions_by_resolve(	×
336	all_dists: AllPythonDistributionTargets,
337	python_setup: PythonSetup,
338	) -> ResolveSortedPythonDistributionTargets:
339	dists = defaultdict(list)	×
340
341	if not python_setup.enable_resolves:	×
342	resolve = None	×
343	return ResolveSortedPythonDistributionTargets(	×
344	FrozenDict({resolve: tuple(all_dists.targets)})
345	)
346
347	dist_owned_deps = await concurrently(	×
348	get_owned_dependencies(DependencyOwner(ExportedTarget(tgt)), **implicitly())
349	for tgt in all_dists.targets
350	)
351
352	for dist, owned_deps in zip(all_dists.targets, dist_owned_deps):	×
353	resolve = None	×
354	# assumption: all owned deps are in the same resolve
355	for dep in owned_deps:	×
356	if dep.target.has_field(PythonResolveField):	×
357	resolve = dep.target[PythonResolveField].normalized_value(python_setup)	×
358	break	×
359	dists[resolve].append(dist)	×
360	return ResolveSortedPythonDistributionTargets(	×
361	FrozenDict({resolve: tuple(sorted(targets)) for resolve, targets in dists.items()})
362	)
363
364
UNCOV 365	@dataclass(frozen=True)	×
UNCOV 366	class EditableLocalDistsRequest:	×
367	"""Request to generate PEP660 wheels of local dists in the given resolve.
368
369	The editable wheel files must not be exported or made available to the end-user (according to
370	PEP 660). Instead, the PEP660 editable wheels serve as intermediate, internal-only,
371	representation of what should be installed in the exported virtualenv to create the editable
372	installs of local python_distributions.
373	"""
374
UNCOV 375	resolve: str \| None # None if resolves is not enabled	×
376
377
UNCOV 378	@dataclass(frozen=True)	×
UNCOV 379	class EditableLocalDists:	×
380	"""A Digest populated by editable (PEP660) wheels of local dists.
381
382	According to PEP660, these wheels should not be exported to users and must be discarded
383	after install. Anything that uses this should ensure that these wheels get installed and
384	then deleted.
385
386	Installing PEP660 wheels creates an "editable" install such that the sys.path gets
387	adjusted to include source directories from the build root (not from the sandbox).
388	This is decidedly not hermetic or portable and should only be used locally.
389
390	PEP660 wheels have .dist-info metadata and the .pth files (or similar) that adjust sys.path.
391	"""
392
UNCOV 393	optional_digest: Digest \| None	×
394
395
UNCOV 396	@rule(desc="Building editable local distributions (PEP 660)")	×
UNCOV 397	async def build_editable_local_dists(	×
398	request: EditableLocalDistsRequest,
399	all_dists: ResolveSortedPythonDistributionTargets,
400	python_setup: PythonSetup,
401	) -> EditableLocalDists:
402	resolve = request.resolve if python_setup.enable_resolves else None	×
403	resolve_dists = all_dists.targets.get(resolve, ())	×
404
405	if not resolve_dists:	×
406	return EditableLocalDists(None)	×
407
408	local_dists_wheels = await concurrently(	×
409	isolate_local_dist_pep660_wheels(target.address, **implicitly()) for target in resolve_dists
410	)
411
412	wheels: list[str] = []	×
413	wheels_digests = []	×
414	for local_dist_wheels in local_dists_wheels:	×
415	wheels.extend(local_dist_wheels.pep660_wheel_paths)	×
416	wheels_digests.append(local_dist_wheels.pep660_wheels_digest)	×
417
418	wheels_digest = await merge_digests(MergeDigests(wheels_digests))	×
419
420	return EditableLocalDists(wheels_digest)	×
421
422
UNCOV 423	def rules():	×
UNCOV 424	return (	×
425	*collect_rules(),
426	*dists_rules(),
427	*package_dists.rules(),
428	*system_binaries.rules(),
429	)

pantsbuild / pants / 19015773527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous