19015773527

Committed 02 Nov 2025 05:33PM UTC coverage: 17.872% (-62.4%) from 80.3%

Build # 19015773527

Build Type

Pull #22816

github

Committed by

web-flow

Commit Message

Merge a12d75757 into 6c024e162

Pull Request Pull Request #22816: Update Pants internal Python to 3.14

Run Details

4 of 5 new or added lines in 3 files covered. (80.0%)

28452 existing lines in 683 files now uncovered.

9831 of 55007 relevant lines covered (17.87%)

0.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/python/pants/backend/python/providers/pyenv/rules.py

1	# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
UNCOV 4	from textwrap import dedent # noqa: PNT20	×
5
UNCOV 6	from pants.backend.python.subsystems.setup import PythonSetup	×
UNCOV 7	from pants.backend.python.util_rules.pex import PythonProvider	×
UNCOV 8	from pants.backend.python.util_rules.pex import rules as pex_rules	×
UNCOV 9	from pants.backend.python.util_rules.pex_environment import PythonExecutable	×
UNCOV 10	from pants.core.goals.resolves import ExportableTool	×
UNCOV 11	from pants.core.goals.run import RunRequest	×
UNCOV 12	from pants.core.util_rules.adhoc_binaries import PythonBuildStandaloneBinary	×
UNCOV 13	from pants.core.util_rules.env_vars import environment_vars_subset	×
UNCOV 14	from pants.core.util_rules.external_tool import TemplatedExternalTool, download_external_tool	×
UNCOV 15	from pants.core.util_rules.external_tool import rules as external_tools_rules	×
UNCOV 16	from pants.engine.env_vars import EXTRA_ENV_VARS_USAGE_HELP, EnvironmentVarsRequest	×
UNCOV 17	from pants.engine.fs import CreateDigest, FileContent	×
UNCOV 18	from pants.engine.internals.native_engine import MergeDigests	×
UNCOV 19	from pants.engine.internals.selectors import concurrently	×
UNCOV 20	from pants.engine.intrinsics import create_digest, merge_digests	×
UNCOV 21	from pants.engine.platform import Platform	×
UNCOV 22	from pants.engine.process import Process, ProcessCacheScope, fallible_to_exec_result_or_raise	×
UNCOV 23	from pants.engine.rules import collect_rules, implicitly, rule	×
UNCOV 24	from pants.engine.unions import UnionRule	×
UNCOV 25	from pants.option.option_types import StrListOption	×
UNCOV 26	from pants.util.frozendict import FrozenDict	×
UNCOV 27	from pants.util.logging import LogLevel	×
UNCOV 28	from pants.util.strutil import softwrap, stable_hash	×
29
UNCOV 30	PYENV_NAMED_CACHE = ".pyenv"	×
UNCOV 31	PYENV_APPEND_ONLY_CACHES = FrozenDict({"pyenv": PYENV_NAMED_CACHE})	×
32
33
UNCOV 34	class PyenvPythonProviderSubsystem(TemplatedExternalTool):	×
UNCOV 35	options_scope = "pyenv-python-provider"	×
UNCOV 36	name = "pyenv"	×
UNCOV 37	help = softwrap(	×
38	"""
39	A subsystem for Pants-provided Python leveraging pyenv (https://github.com/pyenv/pyenv).
40
41	Enabling this subsystem will switch Pants from trying to find an appropriate Python on your
42	system to using pyenv to install the correct Python(s).
43
44	The Pythons provided by Pyenv will be used to run any "user" code (your Python code as well
45	as any Python-based tools you use, like black or pylint). The Pythons are also read-only to
46	ensure they remain hermetic across runs of different tools and code.
47
48	The Pythons themselves are stored in your `named_caches_dir`: https://www.pantsbuild.org/docs/reference-global#named_caches_dir
49	under `pyenv/versions/<version>`. Wiping the relevant version directory (with `sudo rm -rf`)
50	will force a re-install of Python. This may be necessary after changing something about the
51	underlying system which changes the compiled Python, such as installing an
52	optional-at-build-time dependency like `liblzma-dev` (which is used for the optional module
53	`lzma`).
54
55	By default, the subsystem does not pass any optimization flags to the Python compilation
56	process. Doing so would increase the time it takes to install a single Python by about an
57	order of magnitude (E.g. ~2.5 minutes to ~26 minutes).
58	"""
59	)
60
UNCOV 61	default_version = "2.5.3"	×
UNCOV 62	default_url_template = "https://github.com/pyenv/pyenv/archive/refs/tags/v{version}.tar.gz"	×
63
UNCOV 64	class EnvironmentAware:	×
UNCOV 65	installation_extra_env_vars = StrListOption(	×
66	help=softwrap(
67	f"""
68	Additional environment variables to include when running `pyenv install`.
69
70	{EXTRA_ENV_VARS_USAGE_HELP}
71
72	This is especially useful if you want to use an optimized Python (E.g. setting
73	`PYTHON_CONFIGURE_OPTS='--enable-optimizations --with-lto'` and
74	`PYTHON_CFLAGS='-march=native -mtune=native'`) or need custom compiler flags.
75
76	Note that changes to this option result in a different fingerprint for the installed
77	Python, and therefore will cause a full re-install if changed.
78
79	See https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#special-environment-variables
80	for supported env vars.
81	"""
82	),
83	)
84
UNCOV 85	default_known_versions = [	×
86	"2.5.3\|macos_x86_64\|2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e\|1335821",
87	"2.5.3\|macos_arm64\|2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e\|1335821",
88	"2.5.3\|linux_x86_64\|2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e\|1335821",
89	"2.5.3\|linux_arm64\|2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e\|1335821",
90	"2.4.7\|macos_x86_64\|0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d\|799490",
91	"2.4.7\|macos_arm64\|0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d\|799490",
92	"2.4.7\|linux_x86_64\|0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d\|799490",
93	"2.4.7\|linux_arm64\|0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d\|799490",
94	]
95
UNCOV 96	def generate_exe(self, plat: Platform) -> str:	×
97	"""Returns the path to the tool executable.
98
99	If the downloaded artifact is the executable itself, you can leave this unimplemented.
100
101	If the downloaded artifact is an archive, this should be overridden to provide a
102	relative path in the downloaded archive, e.g. `./bin/protoc`.
103	"""
104	return f"./pyenv-{self.version}/bin/pyenv"	×
105
106
UNCOV 107	class PyenvInstallInfoRequest:	×
UNCOV 108	pass	×
109
110
UNCOV 111	@rule	×
UNCOV 112	async def get_pyenv_install_info(	×
113	_: PyenvInstallInfoRequest,
114	pyenv_subsystem: PyenvPythonProviderSubsystem,
115	pyenv_env_aware: PyenvPythonProviderSubsystem.EnvironmentAware,
116	platform: Platform,
117	bootstrap_python: PythonBuildStandaloneBinary,
118	) -> RunRequest:
119	env_vars, pyenv = await concurrently(	×
120	environment_vars_subset(
121	EnvironmentVarsRequest(("PATH",) + pyenv_env_aware.installation_extra_env_vars),
122	**implicitly(),
123	),
124	download_external_tool(pyenv_subsystem.get_request(platform)),
125	)
126	installation_env_vars = {key: name for key, name in env_vars.items() if key != "PATH"}	×
127	installation_fingerprint = stable_hash(installation_env_vars)	×
128	install_script_digest = await create_digest(	×
129	CreateDigest(
130	[
131	# NB: We use a bash script for the hot-path to keep overhead minimal, but a Python
132	# script for the locking+install to be maximally compatible.
133	FileContent(
134	"install_python_shim.sh",
135	dedent(
136	f"""\
137	#!/usr/bin/env bash
138	set -e
139	export PYENV_ROOT=$(readlink {PYENV_NAMED_CACHE})/{installation_fingerprint}
140	DEST="$PYENV_ROOT"/versions/$1
141	if [ ! -f "$DEST"/DONE ]; then
142	mkdir -p "$DEST" 2>/dev/null \|\| true
143	{bootstrap_python.path} install_python_shim.py $1
144	fi
145	echo "$DEST"/bin/python
146	"""
147	).encode(),
148	is_executable=True,
149	),
150	FileContent(
151	"install_python_shim.py",
152	dedent(
153	f"""\
154	import fcntl
155	import pathlib
156	import shutil
157	import subprocess
158	import sys
159
160	PYENV_ROOT = pathlib.Path("{PYENV_NAMED_CACHE}", "{installation_fingerprint}").resolve()
161	SPECIFIC_VERSION = sys.argv[1]
162	SPECIFIC_VERSION_PATH = PYENV_ROOT / "versions" / SPECIFIC_VERSION
163
164	# NB: We put the "DONE" file inside the specific version destination so that
165	# users can wipe the directory clean and expect Pants to re-install that version.
166	DONEFILE_PATH = SPECIFIC_VERSION_PATH / "DONE"
167
168	def main():
169	if DONEFILE_PATH.exists():
170	return
171
172	lockfile_fd = SPECIFIC_VERSION_PATH.with_suffix(".lock").open(mode="w")
173	fcntl.lockf(lockfile_fd, fcntl.LOCK_EX)
174	# Use double-checked locking to ensure that we really need to do the work
175	if DONEFILE_PATH.exists():
176	return
177
178	# If a previous install failed this directory may exist in an intermediate
179	# state, and pyenv may choke trying to install into it, so we remove it.
180	shutil.rmtree(SPECIFIC_VERSION_PATH, ignore_errors=True)
181
182	subprocess.run(["{pyenv.exe}", "install", SPECIFIC_VERSION], check=True)
183	DONEFILE_PATH.touch()
184
185	if __name__ == "__main__":
186	main()
187	"""
188	).encode(),
189	is_executable=True,
190	),
191	]
192	)
193	)
194
195	digest = await merge_digests(MergeDigests([install_script_digest, pyenv.digest]))	×
196	return RunRequest(	×
197	digest=digest,
198	args=["./install_python_shim.sh"],
199	extra_env={
200	"PATH": env_vars.get("PATH", ""),
201	"TMPDIR": "{chroot}/tmpdir",
202	**installation_env_vars,
203	},
204	append_only_caches={PYENV_APPEND_ONLY_CACHES, bootstrap_python.APPEND_ONLY_CACHES},
205	)
206
207
UNCOV 208	class PyenvPythonProvider(PythonProvider):	×
UNCOV 209	pass	×
210
211
UNCOV 212	def _major_minor_patch_to_int(major_minor_patch: str) -> tuple[int, int, int]:	×
213	major, minor, patch = map(int, major_minor_patch.split(".", maxsplit=2))	×
214	return (major, minor, patch)	×
215
216
UNCOV 217	@rule	×
UNCOV 218	async def get_python(	×
219	request: PyenvPythonProvider,
220	python_setup: PythonSetup,
221	platform: Platform,
222	pyenv_subsystem: PyenvPythonProviderSubsystem,
223	) -> PythonExecutable:
224	env_vars, pyenv, pyenv_install = await concurrently(	×
225	environment_vars_subset(EnvironmentVarsRequest(["PATH"]), **implicitly()),
226	download_external_tool(pyenv_subsystem.get_request(platform)),
227	get_pyenv_install_info(PyenvInstallInfoRequest(), **implicitly()),
228	)
229
230	# Determine the lowest major/minor version supported according to the interpreter constraints.
231	major_minor_to_use_str = request.interpreter_constraints.minimum_python_version(	×
232	python_setup.interpreter_versions_universe
233	)
234	if major_minor_to_use_str is None:	×
235	raise ValueError(	×
236	f"Couldn't determine a compatible Interpreter Constraint from {python_setup.interpreter_versions_universe}"
237	)
238
239	# Find the highest patch version given the major/minor version that is known to our version of pyenv.
240	pyenv_latest_known_result = await fallible_to_exec_result_or_raise(	×
241	**implicitly(
242	Process(
243	[pyenv.exe, "latest", "--known", major_minor_to_use_str],
244	input_digest=pyenv.digest,
245	description=f"Choose specific version for Python {major_minor_to_use_str}",
246	env={"PATH": env_vars.get("PATH", "")},
247	)
248	),
249	)
250	major_to_use, minor_to_use, latest_known_patch = _major_minor_patch_to_int(	×
251	pyenv_latest_known_result.stdout.decode().strip()
252	)
253
254	# Pick the highest patch version given the major/minor version that is supported according to
255	# the interpreter constraints and known to our version of pyenv.
256	# We assume pyenv knows every patch version smaller or equal the its latest known patch
257	# version, to avoid calling it for each patch version separately.
258	supported_triplets = request.interpreter_constraints.enumerate_python_versions(	×
259	python_setup.interpreter_versions_universe
260	)
261	try:	×
262	major_minor_patch_to_use = max(	×
263	(major, minor, patch)
264	for (major, minor, patch) in supported_triplets
265	if major == major_to_use and minor == minor_to_use and patch <= latest_known_patch
266	)
267	except ValueError:	×
268	raise ValueError(	×
269	f"Couldn't find a Python {major_minor_to_use_str} version that"
270	f" is compatible with the interpreter constraints {request.interpreter_constraints}"
271	f" and known to pyenv {pyenv_subsystem.version}"
272	f" (latest known version {major_to_use}.{minor_to_use}.{latest_known_patch})."
273	" Suggestion: consider upgrading pyenv or adjusting your interpreter constraints."
274	) from None
275
276	major_minor_patch_to_use_str = ".".join(map(str, major_minor_patch_to_use))	×
277
278	# NB: We don't cache this process at any level for two reasons:
279	# 1. Several tools (including pex) refer to Python at an absolute path, so a named cache is
280	# the only way for this to work reasonably well. Since the named cache could be wiped between
281	# runs (technically during a run, but we can't do anything about that) the
282	# fastest-yet-still-correct solution is to always run this process and make it bail
283	# early-and-quickly if the requisite Python already exists.
284	# 2. Pyenv compiles Python using whatever compiler the system is configured to use. Python
285	# then stores this information so that it can use the same compiler when compiling extension
286	# modules. Therefore caching the compiled Python is somewhat unsafe (especially for a remote
287	# cache). See also https://github.com/pantsbuild/pants/issues/10769.
288	result = await fallible_to_exec_result_or_raise(	×
289	**implicitly(
290	Process(
291	pyenv_install.args + (major_minor_patch_to_use_str,),
292	level=LogLevel.DEBUG,
293	input_digest=pyenv_install.digest,
294	description=f"Install Python {major_minor_patch_to_use_str}",
295	append_only_caches=pyenv_install.append_only_caches,
296	env=pyenv_install.extra_env,
297	# Don't cache, we want this to always be run so that we can assume for the rest of the
298	# session the named_cache destination for this Python is valid, as the Python ecosystem
299	# mainly assumes absolute paths for Python interpreters.
300	cache_scope=ProcessCacheScope.PER_SESSION,
301	)
302	),
303	)
304
305	return PythonExecutable(	×
306	path=result.stdout.decode().splitlines()[-1].strip(),
307	fingerprint=None,
308	append_only_caches=PYENV_APPEND_ONLY_CACHES,
309	)
310
311
UNCOV 312	def rules():	×
UNCOV 313	return (	×
314	*collect_rules(),
315	*pex_rules(),
316	*external_tools_rules(),
317	UnionRule(PythonProvider, PyenvPythonProvider),
318	UnionRule(ExportableTool, PyenvPythonProviderSubsystem),
319	)

pantsbuild / pants / 19015773527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous