18727405282

Committed 22 Oct 2025 07:22PM UTC coverage: 80.276% (+0.01%) from 80.262%

Build # 18727405282

Build Type

push

github

Committed by

web-flow

Commit Message

Enable scoping indexes for specific projects/platforms when generating lockfiles (#22760)

### Problem
The current `generate-lockfiles` goal only allows one locked version for
each requirement. This leads to issues when using certain libraries like
Pytorch and Tensorflow which have some complexities in the way the
wheels are published and tagged. See
https://github.com/pantsbuild/pants/issues/18965

An example with pytorch: the Linux wheels published to PyPI include cuda
dependencies, while the macos ones do not. There is a `+cpu` tag in the
pytorch-hosted index, _but only for Linux_. The current setup means that
we can't lock a cpu-only version of pytorch that works on both macos and
linux.

### Proposed solution
As of [pex 2.56.0](https://github.com/pex-tool/pex/releases/tag/v2.56.0)
it is possible to create a universal lock with two locked resolves (one
for macos, one for linux) by adding an appropriately scoped index:
```bash
pex3 lock create \
    --style universal \
    --target-system linux \
    --target-system mac \
    --elide-unused-requires-dist \
    --interpreter-constraint "CPython==3.13.*" \
    --index pytorch=https://download.pytorch.org/whl/cpu \
    --source "pytorch=torch; sys_platform != 'darwin'" \
    --indent 2 \
    -o lock.json \
    torch
```
This PR adds the option of specifying sources through a new
`[python.resolves_to_sources]` option. Named indexes can already be
defined in `[python-repos.indexes]`

Run Details

94 of 96 new or added lines in 6 files covered. (97.92%)

3 existing lines in 1 file now uncovered.

77860 of 96990 relevant lines covered (80.28%)

3.08 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

81.25

/src/python/pants/testutil/python_interpreter_selection.py

# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import os
import subprocess
from collections.abc import Iterable
from functools import lru_cache
from unittest import skipIf

import _pytest.mark.structures
import pytest

from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints

PY_2 = "2"
PY_3 = "3"

PY_27 = "2.7"
PY_36 = "3.6"
PY_37 = "3.7"
PY_38 = "3.8"
PY_39 = "3.9"
PY_310 = "3.10"
PY_311 = "3.11"


def has_python_version(version):
    """Returns `True` if the current system has the specified version of python.

    :param string version: A python version string, such as 2.7, 3.
    """
    # TODO: Tests that skip unless a python interpreter is present often need the path to that
    # interpreter, and so end up calling python_interpreter_path again. Find a way to streamline this.
    return python_interpreter_path(version) is not None


@lru_cache
def python_interpreter_path(version):
    """Returns the interpreter path if the current system has the specified version of python.

    :param string version: A python version string, such as 2.7, 3.
    :returns: the normalized path to the interpreter binary if found; otherwise `None`
    :rtype: string
    """
    try:
        command = [f"python{version}", "-c", "import sys; print(sys.executable)"]
        py_path = subprocess.check_output(command).decode().strip()
        return os.path.realpath(py_path)
    except (subprocess.CalledProcessError, FileNotFoundError):
        return None


def skip_unless_all_pythons_present(*versions):
    """A decorator that only runs the decorated test method if all of the specified pythons are
    present.

    :param string *versions: Python version strings, such as 2.7, 3.
    """
    missing_versions = [v for v in versions if not has_python_version(v)]
    if len(missing_versions) == 1:
        return skipIf(True, f"Could not find python {missing_versions[0]} on system. Skipping.")
    elif len(missing_versions) > 1:
        return skipIf(
            True,
            "Skipping due to the following missing required pythons: {}".format(
                ", ".join(missing_versions)
            ),
        )
    else:
        return skipIf(False, "All required pythons present, continuing with test!")


def skip_unless_python27_present(func):
    """A test skip decorator that only runs a test method if python2.7 is present."""
    return skip_unless_all_pythons_present(PY_27)(func)


def skip_unless_python3_present(func):
    """A test skip decorator that only runs a test method if python3 is present."""
    return skip_unless_all_pythons_present(PY_3)(func)


def skip_unless_python36_present(func):
    """A test skip decorator that only runs a test method if python3.6 is present."""
    return skip_unless_all_pythons_present(PY_36)(func)


def skip_unless_python37_present(func):
    """A test skip decorator that only runs a test method if python3.7 is present."""
    return skip_unless_all_pythons_present(PY_37)(func)


def skip_unless_python38_present(func):
    """A test skip decorator that only runs a test method if python3.8 is present."""
    return skip_unless_all_pythons_present(PY_38)(func)


def skip_unless_python39_present(func):
    """A test skip decorator that only runs a test method if python3.9 is present."""
    return skip_unless_all_pythons_present(PY_39)(func)


def skip_unless_python310_present(func):
    """A test skip decorator that only runs a test method if python3.10 is present."""
    return skip_unless_all_pythons_present(PY_310)(func)


def skip_unless_python311_present(func):
    """A test skip decorator that only runs a test method if python3.11 is present."""
    return skip_unless_all_pythons_present(PY_311)(func)


def skip_unless_python310_and_python311_present(func):
    """A test skip decorator that only runs a test method if python3.8 and python3.9 are present."""
    return skip_unless_all_pythons_present(PY_310, PY_311)(func)


def skip_unless_python27_and_python3_present(func):
    """A test skip decorator that only runs a test method if python2.7 and python3 are present."""
    return skip_unless_all_pythons_present(PY_27, PY_3)(func)


def skip_unless_python27_and_python36_present(func):
    """A test skip decorator that only runs a test method if python2.7 and python3.6 are present."""
    return skip_unless_all_pythons_present(PY_27, PY_36)(func)


def skip_unless_python38_and_python39_present(func):
    """A test skip decorator that only runs a test method if python3.8 and python3.9 are present."""
    return skip_unless_all_pythons_present(PY_38, PY_39)(func)


def skip_unless_python37_and_python39_present(func):
    """A test skip decorator that only runs a test method if python3.7 and python3.9 are present."""
    return skip_unless_all_pythons_present(PY_37, PY_39)(func)


def all_major_minor_python_versions(
    constraints: Iterable[str],
) -> tuple[_pytest.mark.structures.ParameterSet, ...]:
    """All major.minor Python versions used by the interpreter constraints.

    This is intended to be used with `@pytest.mark.parametrize()` to run a test with every relevant
    Python interpreter.
    """
    versions = InterpreterConstraints(constraints).partition_into_major_minor_versions(
        # Please update this when new stable Python versions are released to CI.
        interpreter_universe=["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
    )
    return tuple(
        pytest.param(
            version,
            marks=pytest.mark.skipif(
                not has_python_version(version),
                reason=f"Could not find python {version} on system. Skipping.",
            ),
        )
        for version in versions
    )

1	# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	10✔
5
6	import os	10✔
7	import subprocess	10✔
8	from collections.abc import Iterable	10✔
9	from functools import lru_cache	10✔
10	from unittest import skipIf	10✔
11
12	import _pytest.mark.structures	10✔
13	import pytest	10✔
14
15	from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints	10✔
16
17	PY_2 = "2"	10✔
18	PY_3 = "3"	10✔
19
20	PY_27 = "2.7"	10✔
21	PY_36 = "3.6"	10✔
22	PY_37 = "3.7"	10✔
23	PY_38 = "3.8"	10✔
24	PY_39 = "3.9"	10✔
25	PY_310 = "3.10"	10✔
26	PY_311 = "3.11"	10✔
27
28
29	def has_python_version(version):	10✔
30	"""Returns `True` if the current system has the specified version of python.
31
32	:param string version: A python version string, such as 2.7, 3.
33	"""
34	# TODO: Tests that skip unless a python interpreter is present often need the path to that
35	# interpreter, and so end up calling python_interpreter_path again. Find a way to streamline this.
36	return python_interpreter_path(version) is not None	10✔
37
38
39	@lru_cache	10✔
40	def python_interpreter_path(version):	10✔
41	"""Returns the interpreter path if the current system has the specified version of python.
42
43	:param string version: A python version string, such as 2.7, 3.
44	:returns: the normalized path to the interpreter binary if found; otherwise `None`
45	:rtype: string
46	"""
47	try:	10✔
48	command = [f"python{version}", "-c", "import sys; print(sys.executable)"]	10✔
49	py_path = subprocess.check_output(command).decode().strip()	10✔
50	return os.path.realpath(py_path)	10✔
UNCOV 51	except (subprocess.CalledProcessError, FileNotFoundError):	×
UNCOV 52	return None	×
53
54
55	def skip_unless_all_pythons_present(*versions):	10✔
56	"""A decorator that only runs the decorated test method if all of the specified pythons are
57	present.
58
59	:param string *versions: Python version strings, such as 2.7, 3.
60	"""
61	missing_versions = [v for v in versions if not has_python_version(v)]	9✔
62	if len(missing_versions) == 1:	9✔
UNCOV 63	return skipIf(True, f"Could not find python {missing_versions[0]} on system. Skipping.")	×
64	elif len(missing_versions) > 1:	9✔
65	return skipIf(	×
66	True,
67	"Skipping due to the following missing required pythons: {}".format(
68	", ".join(missing_versions)
69	),
70	)
71	else:
72	return skipIf(False, "All required pythons present, continuing with test!")	9✔
73
74
75	def skip_unless_python27_present(func):	10✔
76	"""A test skip decorator that only runs a test method if python2.7 is present."""
77	return skip_unless_all_pythons_present(PY_27)(func)	×
78
79
80	def skip_unless_python3_present(func):	10✔
81	"""A test skip decorator that only runs a test method if python3 is present."""
82	return skip_unless_all_pythons_present(PY_3)(func)	×
83
84
85	def skip_unless_python36_present(func):	10✔
86	"""A test skip decorator that only runs a test method if python3.6 is present."""
87	return skip_unless_all_pythons_present(PY_36)(func)	×
88
89
90	def skip_unless_python37_present(func):	10✔
91	"""A test skip decorator that only runs a test method if python3.7 is present."""
92	return skip_unless_all_pythons_present(PY_37)(func)	1✔
93
94
95	def skip_unless_python38_present(func):	10✔
96	"""A test skip decorator that only runs a test method if python3.8 is present."""
97	return skip_unless_all_pythons_present(PY_38)(func)	7✔
98
99
100	def skip_unless_python39_present(func):	10✔
101	"""A test skip decorator that only runs a test method if python3.9 is present."""
102	return skip_unless_all_pythons_present(PY_39)(func)	7✔
103
104
105	def skip_unless_python310_present(func):	10✔
106	"""A test skip decorator that only runs a test method if python3.10 is present."""
107	return skip_unless_all_pythons_present(PY_310)(func)	×
108
109
110	def skip_unless_python311_present(func):	10✔
111	"""A test skip decorator that only runs a test method if python3.11 is present."""
112	return skip_unless_all_pythons_present(PY_311)(func)	×
113
114
115	def skip_unless_python310_and_python311_present(func):	10✔
116	"""A test skip decorator that only runs a test method if python3.8 and python3.9 are present."""
117	return skip_unless_all_pythons_present(PY_310, PY_311)(func)	3✔
118
119
120	def skip_unless_python27_and_python3_present(func):	10✔
121	"""A test skip decorator that only runs a test method if python2.7 and python3 are present."""
122	return skip_unless_all_pythons_present(PY_27, PY_3)(func)	×
123
124
125	def skip_unless_python27_and_python36_present(func):	10✔
126	"""A test skip decorator that only runs a test method if python2.7 and python3.6 are present."""
127	return skip_unless_all_pythons_present(PY_27, PY_36)(func)	×
128
129
130	def skip_unless_python38_and_python39_present(func):	10✔
131	"""A test skip decorator that only runs a test method if python3.8 and python3.9 are present."""
132	return skip_unless_all_pythons_present(PY_38, PY_39)(func)	1✔
133
134
135	def skip_unless_python37_and_python39_present(func):	10✔
136	"""A test skip decorator that only runs a test method if python3.7 and python3.9 are present."""
137	return skip_unless_all_pythons_present(PY_37, PY_39)(func)	×
138
139
140	def all_major_minor_python_versions(	10✔
141	constraints: Iterable[str],
142	) -> tuple[_pytest.mark.structures.ParameterSet, ...]:
143	"""All major.minor Python versions used by the interpreter constraints.
144
145	This is intended to be used with `@pytest.mark.parametrize()` to run a test with every relevant
146	Python interpreter.
147	"""
148	versions = InterpreterConstraints(constraints).partition_into_major_minor_versions(	10✔
149	# Please update this when new stable Python versions are released to CI.
150	interpreter_universe=["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
151	)
152	return tuple(	10✔
153	pytest.param(
154	version,
155	marks=pytest.mark.skipif(
156	not has_python_version(version),
157	reason=f"Could not find python {version} on system. Skipping.",
158	),
159	)
160	for version in versions
161	)

pantsbuild / pants / 18727405282

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous