• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 18812500213

26 Oct 2025 03:42AM UTC coverage: 80.284% (+0.005%) from 80.279%
18812500213

Pull #22804

github

web-flow
Merge 2a56fdb46 into 4834308dc
Pull Request #22804: test_shell_command: use correct default cache scope for a test's environment

29 of 31 new or added lines in 2 files covered. (93.55%)

1314 existing lines in 64 files now uncovered.

77900 of 97030 relevant lines covered (80.28%)

3.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.87
/src/python/pants/core/subsystems/python_bootstrap.py
1
# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
from __future__ import annotations
12✔
5

6
import itertools
12✔
7
import logging
12✔
8
import os
12✔
9
import sys
12✔
10
from collections.abc import Collection
12✔
11
from dataclasses import dataclass
12✔
12

13
from pants.core.environments.target_types import EnvironmentTarget
12✔
14
from pants.core.util_rules import asdf, search_paths
12✔
15
from pants.core.util_rules.asdf import AsdfPathString, AsdfToolPathsResult
12✔
16
from pants.core.util_rules.env_vars import environment_vars_subset
12✔
17
from pants.core.util_rules.search_paths import (
12✔
18
    ValidateSearchPathsRequest,
19
    VersionManagerSearchPathsRequest,
20
    get_un_cachable_version_manager_paths,
21
    validate_search_paths,
22
)
23
from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest, PathEnvironmentVariable
12✔
24
from pants.engine.internals.selectors import concurrently
12✔
25
from pants.engine.rules import collect_rules, implicitly, rule
12✔
26
from pants.option.option_types import DictOption, StrListOption
12✔
27
from pants.option.subsystem import Subsystem
12✔
28
from pants.util.ordered_set import FrozenOrderedSet
12✔
29
from pants.util.strutil import help_text, softwrap
12✔
30

31
logger = logging.getLogger(__name__)
12✔
32

33
_PBS_URL_TEMPLATE = "https://github.com/astral-sh/python-build-standalone/releases/download/20241008/cpython-3.11.10+20241008-{}-install_only.tar.gz"
12✔
34

35

36
class PythonBootstrapSubsystem(Subsystem):
12✔
37
    options_scope = "python-bootstrap"
12✔
38
    help = help_text(
12✔
39
        """
40
        Options used to locate Python interpreters.
41

42
        This subsystem controls where and how Pants will locate Python, but beyond that it does
43
        not control which Python interpreter versions are actually used for your code: see the
44
        `python` subsystem for that.
45
        """
46
    )
47

48
    internal_python_build_standalone_info = DictOption(
12✔
49
        default={
50
            "linux_arm64": (
51
                _PBS_URL_TEMPLATE.format("aarch64-unknown-linux-gnu"),
52
                "320635e957e13d2e10d70a3031563d032fae9e40e60e5ec32bc353643fae1335",
53
                25925875,
54
            ),
55
            "linux_x86_64": (
56
                _PBS_URL_TEMPLATE.format("x86_64-unknown-linux-gnu"),
57
                "ff121f14ed113c9da83a45f76c3cf41976fb4419fe406d5cc7066765761c6a4e",
58
                29716764,
59
            ),
60
            "macos_arm64": (
61
                _PBS_URL_TEMPLATE.format("aarch64-apple-darwin"),
62
                "ecdc9c042b8f97bff211fcf9425bc51c96acd4037df1565964e89816f2c9564d",
63
                17795541,
64
            ),
65
            "macos_x86_64": (
66
                _PBS_URL_TEMPLATE.format("x86_64-apple-darwin"),
67
                "a618c086e0514f681523947e2b66a4dc0c6560f91c36faa072fa6787455df9ea",
68
                18165701,
69
            ),
70
        },
71
        help=softwrap(
72
            """
73
            A map from platform to the information needed to download Python Build Standalone.
74

75
            Python Build Standalone is used to run Python-implemented Pants tools/scripts in
76
            docker environments (so that Python doesn't need to be installed).
77

78
            The version of Python provided should match the default value's version, which is
79
            the highest Python Major/Minor version compatible with the Pants package's
80
            interpreter constraints. Additionally, the downloaded file should be extractable by
81
            `tar` using `-xvf` (most likely a `.tar.gz` file).
82

83
            The schema is `<string platform key>: (<string url>, <string fingerprint>, <int bytelen>)`
84
            for each possible platform.
85
            """
86
        ),
87
        advanced=True,
88
    )
89

90
    class EnvironmentAware(Subsystem.EnvironmentAware):
12✔
91
        search_path = StrListOption(
12✔
92
            default=["<PYENV>", AsdfPathString.STANDARD.value, "<PATH>"],
93
            help=softwrap(
94
                f"""
95
                A list of paths to search for Python interpreters.
96

97
                Which interpreters are actually used from these paths is context-specific:
98
                the Python backend selects interpreters using options on the `python` subsystem,
99
                in particular, the `[python].interpreter_constraints` option.
100

101
                You can specify absolute paths to interpreter binaries
102
                and/or to directories containing interpreter binaries. The order of entries does
103
                not matter.
104

105
                The following special strings are supported:
106

107
                For all runtime environment types:
108

109
                * `<PATH>`, the contents of the PATH env var
110

111
                When the environment is a `local_environment` target:
112

113
                * `{AsdfPathString.STANDARD}`, {AsdfPathString.STANDARD.description("Python")}
114
                * `{AsdfPathString.LOCAL}`, {AsdfPathString.LOCAL.description("interpreter")}
115
                * `<PYENV>`, all Python versions under `$(pyenv root)/versions`
116
                * `<PYENV_LOCAL>`, the Pyenv interpreter with the version in `BUILD_ROOT/.python-version`
117
                * `<PEXRC>`, paths in the `PEX_PYTHON_PATH` variable in `/etc/pexrc` or `~/.pexrc`
118
                """
119
            ),
120
            advanced=True,
121
            metavar="<binary-paths>",
122
        )
123
        names = StrListOption(
12✔
124
            default=["python", "python3"],
125
            help=softwrap(
126
                """
127
                The names of Python binaries to search for. See the `--search-path` option to
128
                influence where interpreters are searched for.
129

130
                This does not impact which Python interpreter is used to run your code, only what
131
                is used to run internal tools.
132
                """
133
            ),
134
            advanced=True,
135
            metavar="<python-binary-names>",
136
        )
137

138

139
@dataclass(frozen=True)
12✔
140
class PythonBootstrap:
12✔
141
    interpreter_names: tuple[str, ...]
12✔
142
    interpreter_search_paths: tuple[str, ...]
12✔
143

144

145
@dataclass(frozen=True)
12✔
146
class _ExpandInterpreterSearchPathsRequest:
12✔
147
    interpreter_search_paths: Collection[str]
12✔
148
    env_tgt: EnvironmentTarget
12✔
149

150

151
@dataclass(frozen=False)
12✔
152
class _SearchPaths:
12✔
153
    paths: tuple[str, ...]
12✔
154

155

156
@rule
12✔
157
async def _expand_interpreter_search_paths(
12✔
158
    request: _ExpandInterpreterSearchPathsRequest, path_env: PathEnvironmentVariable
159
) -> _SearchPaths:
160
    interpreter_search_paths, env_tgt = (request.interpreter_search_paths, request.env_tgt)
×
161

162
    asdf_paths = await AsdfToolPathsResult.get_un_cachable_search_paths(
×
163
        interpreter_search_paths,
164
        env_tgt=env_tgt,
165
        tool_name="python",
166
        tool_description="Python interpreters",
167
        paths_option_name="[python-bootstrap].search_path",
168
    )
169

170
    asdf_standard_tool_paths, asdf_local_tool_paths = (
×
171
        asdf_paths.standard_tool_paths,
172
        asdf_paths.local_tool_paths,
173
    )
174

175
    special_strings = {
×
176
        "<PEXRC>": _get_pex_python_paths,
177
        "<PATH>": lambda: path_env,
178
        AsdfPathString.STANDARD: lambda: asdf_standard_tool_paths,
179
        AsdfPathString.LOCAL: lambda: asdf_local_tool_paths,
180
    }
181

182
    expanded: list[str] = []
×
183
    from_pexrc = None
×
184

185
    pyenv_env = await environment_vars_subset(
×
186
        EnvironmentVarsRequest(("PYENV_ROOT", "HOME")), **implicitly()
187
    )
188
    pyenv_root = _get_pyenv_root(pyenv_env)
×
189
    pyenv_path_results = await concurrently(
×
190
        get_un_cachable_version_manager_paths(
191
            VersionManagerSearchPathsRequest(
192
                env_tgt,
193
                pyenv_root,
194
                "versions",
195
                f"[{PythonBootstrapSubsystem.options_scope}].search_path",
196
                (".python-version",),
197
                s if s == "<PYENV_LOCAL>" else None,
198
            )
199
        )
200
        for s in interpreter_search_paths
201
        if s == "<PYENV>" or s == "<PYENV_LOCAL>"
202
    )
203
    for pyenv_path in FrozenOrderedSet(itertools.chain.from_iterable(pyenv_path_results)):
×
204
        expanded.append(pyenv_path)
×
205
    for s in interpreter_search_paths:
×
206
        if s in special_strings:
×
207
            special_paths = special_strings[s]()
×
208
            if s == "<PEXRC>":
×
209
                from_pexrc = special_paths
×
210
            expanded.extend(special_paths)
×
211
        elif s == "<PYENV>" or s == "<PYENV_LOCAL>":
×
212
            continue
×
213
        else:
214
            expanded.append(s)
×
215
    # Some special-case logging to avoid misunderstandings.
216
    if from_pexrc and len(expanded) > len(from_pexrc):
×
217
        logger.info(
×
218
            softwrap(
219
                f"""
220
                pexrc interpreters requested and found, but other paths were also specified,
221
                so interpreters may not be restricted to the pexrc ones. Full search path is:
222

223
                {":".join(expanded)}
224
                """
225
            )
226
        )
227
    return _SearchPaths(tuple(expanded))
×
228

229

230
# This method is copied from the pex package, located at pex.variables.Variables._get_kv().
231
# It is copied here to avoid a hard dependency on pex.
232
def _get_kv(variable: str) -> list[str] | None:
12✔
UNCOV
233
    kv = variable.strip().split("=")
1✔
UNCOV
234
    if len(list(filter(None, kv))) == 2:
1✔
UNCOV
235
        return kv
1✔
236
    else:
237
        return None
×
238

239

240
# This method is copied from the pex package, located at pex.variables.Variables.from_rc().
241
# It is copied here to avoid a hard dependency on pex.
242
def _read_pex_rc(rc: str | None = None) -> dict[str, str]:
12✔
243
    """Read pex runtime configuration variables from a pexrc file.
244

245
    :param rc: an absolute path to a pexrc file.
246
    :return: A dict of key value pairs found in processed pexrc files.
247
    """
UNCOV
248
    ret_vars = {}
1✔
UNCOV
249
    rc_locations = [
1✔
250
        os.path.join(os.sep, "etc", "pexrc"),
251
        os.path.join("~", ".pexrc"),
252
        os.path.join(os.path.dirname(sys.argv[0]), ".pexrc"),
253
    ]
UNCOV
254
    if rc:
1✔
255
        rc_locations.append(rc)
×
UNCOV
256
    for filename in rc_locations:
1✔
UNCOV
257
        try:
1✔
UNCOV
258
            with open(os.path.expanduser(filename)) as fh:
1✔
UNCOV
259
                rc_items = map(_get_kv, fh)
1✔
UNCOV
260
                ret_vars.update(dict(filter(None, rc_items)))
1✔
UNCOV
261
        except OSError:
1✔
UNCOV
262
            continue
1✔
UNCOV
263
    return ret_vars
1✔
264

265

266
def _get_pex_python_paths():
12✔
267
    """Returns a list of paths to Python interpreters as defined in a pexrc file.
268

269
    These are provided by a PEX_PYTHON_PATH in either of '/etc/pexrc', '~/.pexrc'. PEX_PYTHON_PATH
270
    defines a colon-separated list of paths to interpreters that a pex can be built and run against.
271
    """
UNCOV
272
    ppp = _read_pex_rc().get("PEX_PYTHON_PATH")
1✔
UNCOV
273
    if ppp:
1✔
UNCOV
274
        return ppp.split(os.pathsep)
1✔
275
    else:
276
        return []
×
277

278

279
def _get_pyenv_root(env: EnvironmentVars) -> str | None:
12✔
280
    """See https://github.com/pyenv/pyenv#environment-variables."""
UNCOV
281
    from_env = env.get("PYENV_ROOT")
1✔
UNCOV
282
    if from_env:
1✔
UNCOV
283
        return from_env
1✔
UNCOV
284
    home_from_env = env.get("HOME")
1✔
UNCOV
285
    if home_from_env:
1✔
UNCOV
286
        return os.path.join(home_from_env, ".pyenv")
1✔
UNCOV
287
    return None
1✔
288

289

290
@rule
12✔
291
async def python_bootstrap(
12✔
292
    python_bootstrap_subsystem: PythonBootstrapSubsystem.EnvironmentAware,
293
) -> PythonBootstrap:
294
    interpreter_search_paths = await validate_search_paths(
×
295
        ValidateSearchPathsRequest(
296
            env_tgt=python_bootstrap_subsystem.env_tgt,
297
            search_paths=tuple(python_bootstrap_subsystem.search_path),
298
            option_origin=f"[{PythonBootstrapSubsystem.options_scope}].search_path",
299
            environment_key="python_bootstrap_search_path",
300
            is_default=python_bootstrap_subsystem._is_default("search_path"),
301
            local_only=FrozenOrderedSet(
302
                (
303
                    "<PYENV>",
304
                    "<PYENV_LOCAL>",
305
                    AsdfPathString.STANDARD,
306
                    AsdfPathString.LOCAL,
307
                    "<PEXRC>",
308
                )
309
            ),
310
        )
311
    )
312
    interpreter_names = python_bootstrap_subsystem.names
×
313

314
    expanded_paths = await _expand_interpreter_search_paths(
×
315
        _ExpandInterpreterSearchPathsRequest(
316
            interpreter_search_paths,
317
            python_bootstrap_subsystem.env_tgt,
318
        ),
319
        **implicitly(),
320
    )
321

322
    return PythonBootstrap(
×
323
        interpreter_names=interpreter_names,
324
        interpreter_search_paths=expanded_paths.paths,
325
    )
326

327

328
def rules():
12✔
329
    return (
12✔
330
        *collect_rules(),
331
        *asdf.rules(),
332
        *search_paths.rules(),
333
    )
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc