22285099215

Committed 22 Feb 2026 08:52PM UTC coverage: 75.854% (-17.1%) from 92.936%

Build # 22285099215

Build Type

Pull #23121

github

Committed by

web-flow

Commit Message

Merge c7299df9c into ba8359840

Pull Request Pull Request #23121: fix issue with optional fields in dependency validator

Run Details

28 of 29 new or added lines in 2 files covered. (96.55%)

11174 existing lines in 400 files now uncovered.

53694 of 70786 relevant lines covered (75.85%)

1.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.71

/src/python/pants/backend/python/providers/pyenv/rules.py

# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from textwrap import dedent  # noqa: PNT20

from pants.backend.python.subsystems.setup import PythonSetup
from pants.backend.python.util_rules.pex import PythonProvider
from pants.backend.python.util_rules.pex import rules as pex_rules
from pants.backend.python.util_rules.pex_environment import PythonExecutable
from pants.core.goals.resolves import ExportableTool
from pants.core.goals.run import RunRequest
from pants.core.util_rules.adhoc_binaries import PythonBuildStandaloneBinary
from pants.core.util_rules.env_vars import environment_vars_subset
from pants.core.util_rules.external_tool import TemplatedExternalTool, download_external_tool
from pants.core.util_rules.external_tool import rules as external_tools_rules
from pants.engine.env_vars import EXTRA_ENV_VARS_USAGE_HELP, EnvironmentVarsRequest
from pants.engine.fs import CreateDigest, FileContent
from pants.engine.internals.native_engine import MergeDigests
from pants.engine.internals.selectors import concurrently
from pants.engine.intrinsics import create_digest, merge_digests
from pants.engine.platform import Platform
from pants.engine.process import Process, ProcessCacheScope, fallible_to_exec_result_or_raise
from pants.engine.rules import collect_rules, implicitly, rule
from pants.engine.unions import UnionRule
from pants.option.option_types import StrListOption
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel
from pants.util.strutil import softwrap, stable_hash

PYENV_NAMED_CACHE = ".pyenv"
PYENV_APPEND_ONLY_CACHES = FrozenDict({"pyenv": PYENV_NAMED_CACHE})


class PyenvPythonProviderSubsystem(TemplatedExternalTool):
    options_scope = "pyenv-python-provider"
    name = "pyenv"
    help = softwrap(
        """
        A subsystem for Pants-provided Python leveraging pyenv (https://github.com/pyenv/pyenv).

        Enabling this subsystem will switch Pants from trying to find an appropriate Python on your
        system to using pyenv to install the correct Python(s).

        The Pythons provided by Pyenv will be used to run any "user" code (your Python code as well
        as any Python-based tools you use, like black or pylint). The Pythons are also read-only to
        ensure they remain hermetic across runs of different tools and code.

        The Pythons themselves are stored in your `named_caches_dir`: https://www.pantsbuild.org/docs/reference-global#named_caches_dir
        under `pyenv/versions/<version>`. Wiping the relevant version directory (with `sudo rm -rf`)
        will force a re-install of Python. This may be necessary after changing something about the
        underlying system which changes the compiled Python, such as installing an
        optional-at-build-time dependency like `liblzma-dev` (which is used for the optional module
        `lzma`).

        By default, the subsystem does not pass any optimization flags to the Python compilation
        process. Doing so would increase the time it takes to install a single Python by about an
        order of magnitude (E.g. ~2.5 minutes to ~26 minutes).
        """
    )

    default_version = "2.5.3"
    default_url_template = "https://github.com/pyenv/pyenv/archive/refs/tags/v{version}.tar.gz"

    class EnvironmentAware:
        installation_extra_env_vars = StrListOption(
            help=softwrap(
                f"""
                Additional environment variables to include when running `pyenv install`.

                {EXTRA_ENV_VARS_USAGE_HELP}

                This is especially useful if you want to use an optimized Python (E.g. setting
                `PYTHON_CONFIGURE_OPTS='--enable-optimizations --with-lto'` and
                `PYTHON_CFLAGS='-march=native -mtune=native'`) or need custom compiler flags.

                Note that changes to this option result in a different fingerprint for the installed
                Python, and therefore will cause a full re-install if changed.

                See https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#special-environment-variables
                for supported env vars.
                """
            ),
        )

    default_known_versions = [
        "2.5.3|linux_arm64 |2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e|1335821",
        "2.5.3|linux_x86_64|2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e|1335821",
        "2.5.3|macos_arm64 |2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e|1335821",
        "2.5.3|macos_x86_64|2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e|1335821",
        "2.4.7|linux_arm64 |0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d|799490",
        "2.4.7|linux_x86_64|0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d|799490",
        "2.4.7|macos_arm64 |0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d|799490",
        "2.4.7|macos_x86_64|0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d|799490",
    ]

    def generate_exe(self, plat: Platform) -> str:
        """Returns the path to the tool executable.

        If the downloaded artifact is the executable itself, you can leave this unimplemented.

        If the downloaded artifact is an archive, this should be overridden to provide a
        relative path in the downloaded archive, e.g. `./bin/protoc`.
        """
        return f"./pyenv-{self.version}/bin/pyenv"


class PyenvInstallInfoRequest:
    pass


@rule
async def get_pyenv_install_info(
    _: PyenvInstallInfoRequest,
    pyenv_subsystem: PyenvPythonProviderSubsystem,
    pyenv_env_aware: PyenvPythonProviderSubsystem.EnvironmentAware,
    platform: Platform,
    bootstrap_python: PythonBuildStandaloneBinary,
) -> RunRequest:
    env_vars, pyenv = await concurrently(
        environment_vars_subset(
            EnvironmentVarsRequest(("PATH",) + pyenv_env_aware.installation_extra_env_vars),
            **implicitly(),
        ),
        download_external_tool(pyenv_subsystem.get_request(platform)),
    )
    installation_env_vars = {key: name for key, name in env_vars.items() if key != "PATH"}
    installation_fingerprint = stable_hash(installation_env_vars)
    install_script_digest = await create_digest(
        CreateDigest(
            [
                # NB: We use a bash script for the hot-path to keep overhead minimal, but a Python
                # script for the locking+install to be maximally compatible.
                FileContent(
                    "install_python_shim.sh",
                    dedent(
                        f"""\
                        #!/usr/bin/env bash
                        set -e
                        export PYENV_ROOT=$(readlink {PYENV_NAMED_CACHE})/{installation_fingerprint}
                        DEST="$PYENV_ROOT"/versions/$1
                        if [ ! -f "$DEST"/DONE ]; then
                            mkdir -p "$DEST" 2>/dev/null || true
                            {bootstrap_python.path} install_python_shim.py $1
                        fi
                        echo "$DEST"/bin/python
                        """
                    ).encode(),
                    is_executable=True,
                ),
                FileContent(
                    "install_python_shim.py",
                    dedent(
                        f"""\
                        import fcntl
                        import pathlib
                        import shutil
                        import subprocess
                        import sys

                        PYENV_ROOT = pathlib.Path("{PYENV_NAMED_CACHE}", "{installation_fingerprint}").resolve()
                        SPECIFIC_VERSION = sys.argv[1]
                        SPECIFIC_VERSION_PATH = PYENV_ROOT / "versions" / SPECIFIC_VERSION

                        # NB: We put the "DONE" file inside the specific version destination so that
                        # users can wipe the directory clean and expect Pants to re-install that version.
                        DONEFILE_PATH = SPECIFIC_VERSION_PATH / "DONE"

                        def main():
                            if DONEFILE_PATH.exists():
                                return

                            lockfile_fd = SPECIFIC_VERSION_PATH.with_suffix(".lock").open(mode="w")
                            fcntl.lockf(lockfile_fd, fcntl.LOCK_EX)
                            # Use double-checked locking to ensure that we really need to do the work
                            if DONEFILE_PATH.exists():
                                return

                            # If a previous install failed this directory may exist in an intermediate
                            # state, and pyenv may choke trying to install into it, so we remove it.
                            shutil.rmtree(SPECIFIC_VERSION_PATH, ignore_errors=True)

                            subprocess.run(["{pyenv.exe}", "install", SPECIFIC_VERSION], check=True)
                            DONEFILE_PATH.touch()

                        if __name__ == "__main__":
                            main()
                        """
                    ).encode(),
                    is_executable=True,
                ),
            ]
        )
    )

    digest = await merge_digests(MergeDigests([install_script_digest, pyenv.digest]))
    return RunRequest(
        digest=digest,
        args=["./install_python_shim.sh"],
        extra_env={
            "PATH": env_vars.get("PATH", ""),
            "TMPDIR": "{chroot}/tmpdir",
            **installation_env_vars,
        },
        append_only_caches={**PYENV_APPEND_ONLY_CACHES, **bootstrap_python.APPEND_ONLY_CACHES},
    )


class PyenvPythonProvider(PythonProvider):
    pass


def _major_minor_patch_to_int(major_minor_patch: str) -> tuple[int, int, int]:
    major, minor, patch = map(int, major_minor_patch.split(".", maxsplit=2))
    return (major, minor, patch)


@rule
async def get_python(
    request: PyenvPythonProvider,
    python_setup: PythonSetup,
    platform: Platform,
    pyenv_subsystem: PyenvPythonProviderSubsystem,
) -> PythonExecutable:
    env_vars, pyenv, pyenv_install = await concurrently(
        environment_vars_subset(EnvironmentVarsRequest(["PATH"]), **implicitly()),
        download_external_tool(pyenv_subsystem.get_request(platform)),
        get_pyenv_install_info(PyenvInstallInfoRequest(), **implicitly()),
    )

    # Determine the lowest major/minor version supported according to the interpreter constraints.
    major_minor_to_use_str = request.interpreter_constraints.minimum_python_version(
        python_setup.interpreter_versions_universe
    )
    if major_minor_to_use_str is None:
        raise ValueError(
            f"Couldn't determine a compatible Interpreter Constraint from {python_setup.interpreter_versions_universe}"
        )

    # Find the highest patch version given the major/minor version that is known to our version of pyenv.
    pyenv_latest_known_result = await fallible_to_exec_result_or_raise(
        **implicitly(
            Process(
                [pyenv.exe, "latest", "--known", major_minor_to_use_str],
                input_digest=pyenv.digest,
                description=f"Choose specific version for Python {major_minor_to_use_str}",
                env={"PATH": env_vars.get("PATH", "")},
            )
        ),
    )
    major_to_use, minor_to_use, latest_known_patch = _major_minor_patch_to_int(
        pyenv_latest_known_result.stdout.decode().strip()
    )

    # Pick the highest patch version given the major/minor version that is supported according to
    # the interpreter constraints and known to our version of pyenv.
    # We assume pyenv knows every patch version smaller or equal the its latest known patch
    # version, to avoid calling it for each patch version separately.
    supported_triplets = request.interpreter_constraints.enumerate_python_versions(
        python_setup.interpreter_versions_universe
    )
    try:
        major_minor_patch_to_use = max(
            (major, minor, patch)
            for (major, minor, patch) in supported_triplets
            if major == major_to_use and minor == minor_to_use and patch <= latest_known_patch
        )
    except ValueError:
        raise ValueError(
            f"Couldn't find a Python {major_minor_to_use_str} version that"
            f" is compatible with the interpreter constraints {request.interpreter_constraints}"
            f" and known to pyenv {pyenv_subsystem.version}"
            f" (latest known version {major_to_use}.{minor_to_use}.{latest_known_patch})."
            " Suggestion: consider upgrading pyenv or adjusting your interpreter constraints."
        ) from None

    major_minor_patch_to_use_str = ".".join(map(str, major_minor_patch_to_use))

    # NB: We don't cache this process at any level for two reasons:
    #   1. Several tools (including pex) refer to Python at an absolute path, so a named cache is
    #   the only way for this to work reasonably well. Since the named cache could be wiped between
    #   runs (technically during a run, but we can't do anything about that) the
    #   fastest-yet-still-correct solution is to always run this process and make it bail
    #   early-and-quickly if the requisite Python already exists.
    #   2. Pyenv compiles Python using whatever compiler the system is configured to use. Python
    #   then stores this information so that it can use the same compiler when compiling extension
    #   modules. Therefore caching the compiled Python is somewhat unsafe (especially for a remote
    #   cache). See also https://github.com/pantsbuild/pants/issues/10769.
    result = await fallible_to_exec_result_or_raise(
        **implicitly(
            Process(
                pyenv_install.args + (major_minor_patch_to_use_str,),
                level=LogLevel.DEBUG,
                input_digest=pyenv_install.digest,
                description=f"Install Python {major_minor_patch_to_use_str}",
                append_only_caches=pyenv_install.append_only_caches,
                env=pyenv_install.extra_env,
                # Don't cache, we want this to always be run so that we can assume for the rest of the
                # session the named_cache destination for this Python is valid, as the Python ecosystem
                # mainly assumes absolute paths for Python interpreters.
                cache_scope=ProcessCacheScope.PER_SESSION,
            )
        ),
    )

    return PythonExecutable(
        path=result.stdout.decode().splitlines()[-1].strip(),
        fingerprint=None,
        append_only_caches=PYENV_APPEND_ONLY_CACHES,
    )


def rules():
    return (
        *collect_rules(),
        *pex_rules(),
        *external_tools_rules(),
        UnionRule(PythonProvider, PyenvPythonProvider),
        UnionRule(ExportableTool, PyenvPythonProviderSubsystem),
    )

1	# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from textwrap import dedent # noqa: PNT20	1✔
5
6	from pants.backend.python.subsystems.setup import PythonSetup	1✔
7	from pants.backend.python.util_rules.pex import PythonProvider	1✔
8	from pants.backend.python.util_rules.pex import rules as pex_rules	1✔
9	from pants.backend.python.util_rules.pex_environment import PythonExecutable	1✔
10	from pants.core.goals.resolves import ExportableTool	1✔
11	from pants.core.goals.run import RunRequest	1✔
12	from pants.core.util_rules.adhoc_binaries import PythonBuildStandaloneBinary	1✔
13	from pants.core.util_rules.env_vars import environment_vars_subset	1✔
14	from pants.core.util_rules.external_tool import TemplatedExternalTool, download_external_tool	1✔
15	from pants.core.util_rules.external_tool import rules as external_tools_rules	1✔
16	from pants.engine.env_vars import EXTRA_ENV_VARS_USAGE_HELP, EnvironmentVarsRequest	1✔
17	from pants.engine.fs import CreateDigest, FileContent	1✔
18	from pants.engine.internals.native_engine import MergeDigests	1✔
19	from pants.engine.internals.selectors import concurrently	1✔
20	from pants.engine.intrinsics import create_digest, merge_digests	1✔
21	from pants.engine.platform import Platform	1✔
22	from pants.engine.process import Process, ProcessCacheScope, fallible_to_exec_result_or_raise	1✔
23	from pants.engine.rules import collect_rules, implicitly, rule	1✔
24	from pants.engine.unions import UnionRule	1✔
25	from pants.option.option_types import StrListOption	1✔
26	from pants.util.frozendict import FrozenDict	1✔
27	from pants.util.logging import LogLevel	1✔
28	from pants.util.strutil import softwrap, stable_hash	1✔
29
30	PYENV_NAMED_CACHE = ".pyenv"	1✔
31	PYENV_APPEND_ONLY_CACHES = FrozenDict({"pyenv": PYENV_NAMED_CACHE})	1✔
32
33
34	class PyenvPythonProviderSubsystem(TemplatedExternalTool):	1✔
35	options_scope = "pyenv-python-provider"	1✔
36	name = "pyenv"	1✔
37	help = softwrap(	1✔
38	"""
39	A subsystem for Pants-provided Python leveraging pyenv (https://github.com/pyenv/pyenv).
40
41	Enabling this subsystem will switch Pants from trying to find an appropriate Python on your
42	system to using pyenv to install the correct Python(s).
43
44	The Pythons provided by Pyenv will be used to run any "user" code (your Python code as well
45	as any Python-based tools you use, like black or pylint). The Pythons are also read-only to
46	ensure they remain hermetic across runs of different tools and code.
47
48	The Pythons themselves are stored in your `named_caches_dir`: https://www.pantsbuild.org/docs/reference-global#named_caches_dir
49	under `pyenv/versions/<version>`. Wiping the relevant version directory (with `sudo rm -rf`)
50	will force a re-install of Python. This may be necessary after changing something about the
51	underlying system which changes the compiled Python, such as installing an
52	optional-at-build-time dependency like `liblzma-dev` (which is used for the optional module
53	`lzma`).
54
55	By default, the subsystem does not pass any optimization flags to the Python compilation
56	process. Doing so would increase the time it takes to install a single Python by about an
57	order of magnitude (E.g. ~2.5 minutes to ~26 minutes).
58	"""
59	)
60
61	default_version = "2.5.3"	1✔
62	default_url_template = "https://github.com/pyenv/pyenv/archive/refs/tags/v{version}.tar.gz"	1✔
63
64	class EnvironmentAware:	1✔
65	installation_extra_env_vars = StrListOption(	1✔
66	help=softwrap(
67	f"""
68	Additional environment variables to include when running `pyenv install`.
69
70	{EXTRA_ENV_VARS_USAGE_HELP}
71
72	This is especially useful if you want to use an optimized Python (E.g. setting
73	`PYTHON_CONFIGURE_OPTS='--enable-optimizations --with-lto'` and
74	`PYTHON_CFLAGS='-march=native -mtune=native'`) or need custom compiler flags.
75
76	Note that changes to this option result in a different fingerprint for the installed
77	Python, and therefore will cause a full re-install if changed.
78
79	See https://github.com/pyenv/pyenv/blob/master/plugins/python-build/README.md#special-environment-variables
80	for supported env vars.
81	"""
82	),
83	)
84
85	default_known_versions = [	1✔
86	"2.5.3\|linux_arm64 \|2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e\|1335821",
87	"2.5.3\|linux_x86_64\|2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e\|1335821",
88	"2.5.3\|macos_arm64 \|2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e\|1335821",
89	"2.5.3\|macos_x86_64\|2068872d4f3174d697bcfd602ada3dc2b7764e84f73be7850c0de86fbf00f69e\|1335821",
90	"2.4.7\|linux_arm64 \|0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d\|799490",
91	"2.4.7\|linux_x86_64\|0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d\|799490",
92	"2.4.7\|macos_arm64 \|0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d\|799490",
93	"2.4.7\|macos_x86_64\|0c0137963dd3c4b356663a3a152a64815e5e4364f131f2976a2731a13ab1de4d\|799490",
94	]
95
96	def generate_exe(self, plat: Platform) -> str:	1✔
97	"""Returns the path to the tool executable.
98
99	If the downloaded artifact is the executable itself, you can leave this unimplemented.
100
101	If the downloaded artifact is an archive, this should be overridden to provide a
102	relative path in the downloaded archive, e.g. `./bin/protoc`.
103	"""
104	return f"./pyenv-{self.version}/bin/pyenv"	1✔
105
106
107	class PyenvInstallInfoRequest:	1✔
108	pass	1✔
109
110
111	@rule	1✔
112	async def get_pyenv_install_info(	1✔
113	_: PyenvInstallInfoRequest,
114	pyenv_subsystem: PyenvPythonProviderSubsystem,
115	pyenv_env_aware: PyenvPythonProviderSubsystem.EnvironmentAware,
116	platform: Platform,
117	bootstrap_python: PythonBuildStandaloneBinary,
118	) -> RunRequest:
119	env_vars, pyenv = await concurrently(	1✔
120	environment_vars_subset(
121	EnvironmentVarsRequest(("PATH",) + pyenv_env_aware.installation_extra_env_vars),
122	**implicitly(),
123	),
124	download_external_tool(pyenv_subsystem.get_request(platform)),
125	)
126	installation_env_vars = {key: name for key, name in env_vars.items() if key != "PATH"}	1✔
127	installation_fingerprint = stable_hash(installation_env_vars)	1✔
128	install_script_digest = await create_digest(	1✔
129	CreateDigest(
130	[
131	# NB: We use a bash script for the hot-path to keep overhead minimal, but a Python
132	# script for the locking+install to be maximally compatible.
133	FileContent(
134	"install_python_shim.sh",
135	dedent(
136	f"""\
137	#!/usr/bin/env bash
138	set -e
139	export PYENV_ROOT=$(readlink {PYENV_NAMED_CACHE})/{installation_fingerprint}
140	DEST="$PYENV_ROOT"/versions/$1
141	if [ ! -f "$DEST"/DONE ]; then
142	mkdir -p "$DEST" 2>/dev/null \|\| true
143	{bootstrap_python.path} install_python_shim.py $1
144	fi
145	echo "$DEST"/bin/python
146	"""
147	).encode(),
148	is_executable=True,
149	),
150	FileContent(
151	"install_python_shim.py",
152	dedent(
153	f"""\
154	import fcntl
155	import pathlib
156	import shutil
157	import subprocess
158	import sys
159
160	PYENV_ROOT = pathlib.Path("{PYENV_NAMED_CACHE}", "{installation_fingerprint}").resolve()
161	SPECIFIC_VERSION = sys.argv[1]
162	SPECIFIC_VERSION_PATH = PYENV_ROOT / "versions" / SPECIFIC_VERSION
163
164	# NB: We put the "DONE" file inside the specific version destination so that
165	# users can wipe the directory clean and expect Pants to re-install that version.
166	DONEFILE_PATH = SPECIFIC_VERSION_PATH / "DONE"
167
168	def main():
169	if DONEFILE_PATH.exists():
170	return
171
172	lockfile_fd = SPECIFIC_VERSION_PATH.with_suffix(".lock").open(mode="w")
173	fcntl.lockf(lockfile_fd, fcntl.LOCK_EX)
174	# Use double-checked locking to ensure that we really need to do the work
175	if DONEFILE_PATH.exists():
176	return
177
178	# If a previous install failed this directory may exist in an intermediate
179	# state, and pyenv may choke trying to install into it, so we remove it.
180	shutil.rmtree(SPECIFIC_VERSION_PATH, ignore_errors=True)
181
182	subprocess.run(["{pyenv.exe}", "install", SPECIFIC_VERSION], check=True)
183	DONEFILE_PATH.touch()
184
185	if __name__ == "__main__":
186	main()
187	"""
188	).encode(),
189	is_executable=True,
190	),
191	]
192	)
193	)
194
195	digest = await merge_digests(MergeDigests([install_script_digest, pyenv.digest]))	1✔
196	return RunRequest(	1✔
197	digest=digest,
198	args=["./install_python_shim.sh"],
199	extra_env={
200	"PATH": env_vars.get("PATH", ""),
201	"TMPDIR": "{chroot}/tmpdir",
202	**installation_env_vars,
203	},
204	append_only_caches={PYENV_APPEND_ONLY_CACHES, bootstrap_python.APPEND_ONLY_CACHES},
205	)
206
207
208	class PyenvPythonProvider(PythonProvider):	1✔
209	pass	1✔
210
211
212	def _major_minor_patch_to_int(major_minor_patch: str) -> tuple[int, int, int]:	1✔
213	major, minor, patch = map(int, major_minor_patch.split(".", maxsplit=2))	1✔
214	return (major, minor, patch)	1✔
215
216
217	@rule	1✔
218	async def get_python(	1✔
219	request: PyenvPythonProvider,
220	python_setup: PythonSetup,
221	platform: Platform,
222	pyenv_subsystem: PyenvPythonProviderSubsystem,
223	) -> PythonExecutable:
224	env_vars, pyenv, pyenv_install = await concurrently(	1✔
225	environment_vars_subset(EnvironmentVarsRequest(["PATH"]), **implicitly()),
226	download_external_tool(pyenv_subsystem.get_request(platform)),
227	get_pyenv_install_info(PyenvInstallInfoRequest(), **implicitly()),
228	)
229
230	# Determine the lowest major/minor version supported according to the interpreter constraints.
231	major_minor_to_use_str = request.interpreter_constraints.minimum_python_version(	1✔
232	python_setup.interpreter_versions_universe
233	)
234	if major_minor_to_use_str is None:	1✔
235	raise ValueError(	×
236	f"Couldn't determine a compatible Interpreter Constraint from {python_setup.interpreter_versions_universe}"
237	)
238
239	# Find the highest patch version given the major/minor version that is known to our version of pyenv.
240	pyenv_latest_known_result = await fallible_to_exec_result_or_raise(	1✔
241	**implicitly(
242	Process(
243	[pyenv.exe, "latest", "--known", major_minor_to_use_str],
244	input_digest=pyenv.digest,
245	description=f"Choose specific version for Python {major_minor_to_use_str}",
246	env={"PATH": env_vars.get("PATH", "")},
247	)
248	),
249	)
250	major_to_use, minor_to_use, latest_known_patch = _major_minor_patch_to_int(	1✔
251	pyenv_latest_known_result.stdout.decode().strip()
252	)
253
254	# Pick the highest patch version given the major/minor version that is supported according to
255	# the interpreter constraints and known to our version of pyenv.
256	# We assume pyenv knows every patch version smaller or equal the its latest known patch
257	# version, to avoid calling it for each patch version separately.
258	supported_triplets = request.interpreter_constraints.enumerate_python_versions(	1✔
259	python_setup.interpreter_versions_universe
260	)
261	try:	1✔
262	major_minor_patch_to_use = max(	1✔
263	(major, minor, patch)
264	for (major, minor, patch) in supported_triplets
265	if major == major_to_use and minor == minor_to_use and patch <= latest_known_patch
266	)
UNCOV 267	except ValueError:	×
UNCOV 268	raise ValueError(	×
269	f"Couldn't find a Python {major_minor_to_use_str} version that"
270	f" is compatible with the interpreter constraints {request.interpreter_constraints}"
271	f" and known to pyenv {pyenv_subsystem.version}"
272	f" (latest known version {major_to_use}.{minor_to_use}.{latest_known_patch})."
273	" Suggestion: consider upgrading pyenv or adjusting your interpreter constraints."
274	) from None
275
276	major_minor_patch_to_use_str = ".".join(map(str, major_minor_patch_to_use))	1✔
277
278	# NB: We don't cache this process at any level for two reasons:
279	# 1. Several tools (including pex) refer to Python at an absolute path, so a named cache is
280	# the only way for this to work reasonably well. Since the named cache could be wiped between
281	# runs (technically during a run, but we can't do anything about that) the
282	# fastest-yet-still-correct solution is to always run this process and make it bail
283	# early-and-quickly if the requisite Python already exists.
284	# 2. Pyenv compiles Python using whatever compiler the system is configured to use. Python
285	# then stores this information so that it can use the same compiler when compiling extension
286	# modules. Therefore caching the compiled Python is somewhat unsafe (especially for a remote
287	# cache). See also https://github.com/pantsbuild/pants/issues/10769.
288	result = await fallible_to_exec_result_or_raise(	1✔
289	**implicitly(
290	Process(
291	pyenv_install.args + (major_minor_patch_to_use_str,),
292	level=LogLevel.DEBUG,
293	input_digest=pyenv_install.digest,
294	description=f"Install Python {major_minor_patch_to_use_str}",
295	append_only_caches=pyenv_install.append_only_caches,
296	env=pyenv_install.extra_env,
297	# Don't cache, we want this to always be run so that we can assume for the rest of the
298	# session the named_cache destination for this Python is valid, as the Python ecosystem
299	# mainly assumes absolute paths for Python interpreters.
300	cache_scope=ProcessCacheScope.PER_SESSION,
301	)
302	),
303	)
304
305	return PythonExecutable(	1✔
306	path=result.stdout.decode().splitlines()[-1].strip(),
307	fingerprint=None,
308	append_only_caches=PYENV_APPEND_ONLY_CACHES,
309	)
310
311
312	def rules():	1✔
313	return (	1✔
314	*collect_rules(),
315	*pex_rules(),
316	*external_tools_rules(),
317	UnionRule(PythonProvider, PyenvPythonProvider),
318	UnionRule(ExportableTool, PyenvPythonProviderSubsystem),
319	)

pantsbuild / pants / 22285099215

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous