18847018991

Committed 27 Oct 2025 03:45PM UTC coverage: 92.254% (+12.0%) from 80.282%

Build # 18847018991

Build Type

Pull #22816

github

Committed by

web-flow

Commit Message

Merge f1312fa87 into 06e216752

Pull Request Pull Request #22816: Update Pants internal Python to 3.14

Run Details

39 of 40 new or added lines in 11 files covered. (97.5%)

382 existing lines in 22 files now uncovered.

89230 of 96722 relevant lines covered (92.25%)

3.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.15

/src/python/pants/backend/python/goals/export_test.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
import dataclasses
import os
import re
import sys
from textwrap import dedent

import pytest

from pants.backend.python import target_types_rules
from pants.backend.python.goals import export
from pants.backend.python.goals.export import ExportVenvsRequest, PythonResolveExportFormat
from pants.backend.python.lint.isort import subsystem as isort_subsystem
from pants.backend.python.macros.python_artifact import PythonArtifact
from pants.backend.python.target_types import (
    PythonDistribution,
    PythonRequirementTarget,
    PythonResolveField,
    PythonSourceField,
    PythonSourcesGeneratorTarget,
)
from pants.backend.python.util_rules import local_dists_pep660, pex_from_targets
from pants.base.specs import RawSpecs
from pants.core.goals.export import ExportResults
from pants.core.util_rules import distdir
from pants.engine.fs import CreateDigest
from pants.engine.internals.native_engine import Snapshot
from pants.engine.internals.parametrize import Parametrize
from pants.engine.intrinsics import digest_to_snapshot, get_digest_contents
from pants.engine.rules import QueryRule, implicitly, rule
from pants.engine.target import (
    GeneratedSources,
    GenerateSourcesRequest,
    SingleSourceField,
    Target,
    Targets,
)
from pants.engine.unions import UnionRule
from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, RuleRunner
from pants.util.frozendict import FrozenDict

pants_args_for_python_lockfiles = [
    "--python-enable-resolves=True",
    # Turn off lockfile validation to make the test simpler.
    "--python-invalid-lockfile-behavior=ignore",
    # Turn off python synthetic lockfile targets to make the test simpler.
    "--no-python-enable-lockfile-targets",
]


@pytest.fixture
def rule_runner() -> RuleRunner:
    return RuleRunner(
        rules=[
            *export.rules(),
            *pex_from_targets.rules(),
            *target_types_rules.rules(),
            *distdir.rules(),
            *local_dists_pep660.rules(),
            *isort_subsystem.rules(),  # add a tool that we can try exporting
            QueryRule(Targets, [RawSpecs]),
            QueryRule(ExportResults, [ExportVenvsRequest]),
        ],
        target_types=[PythonRequirementTarget, PythonSourcesGeneratorTarget, PythonDistribution],
        objects={"python_artifact": PythonArtifact, "parametrize": Parametrize},
    )


@pytest.mark.parametrize(
    "py_resolve_format,py_hermetic_scripts",
    [
        (PythonResolveExportFormat.symlinked_immutable_virtualenv, True),
        (PythonResolveExportFormat.mutable_virtualenv, True),
        (PythonResolveExportFormat.mutable_virtualenv, False),
    ],
)
def test_export_venv_new_codepath(
    rule_runner: RuleRunner,
    py_resolve_format: PythonResolveExportFormat,
    py_hermetic_scripts: bool,
) -> None:
    # We know that the current interpreter exists on the system.
    vinfo = sys.version_info
    current_interpreter = f"{vinfo.major}.{vinfo.minor}.{vinfo.micro}"
    rule_runner.write_files(
        {
            "src/foo/__init__.py": "from colors import *",
            "src/foo/BUILD": dedent(
                """\
                python_sources(name='foo', resolve=parametrize('a', 'b'))
                python_distribution(
                    name='dist',
                    provides=python_artifact(name='foo', version='1.2.3'),
                    dependencies=[':foo@resolve=a'],
                )
                python_requirement(name='req1', requirements=['ansicolors==1.1.8'], resolve='a')
                python_requirement(name='req2', requirements=['ansicolors==1.1.8'], resolve='b')
                """
            ),
            "lock.txt": "ansicolors==1.1.8",
        }
    )

    format_flag = f"--export-py-resolve-format={py_resolve_format.value}"
    hermetic_flags = (
        [] if py_hermetic_scripts else ["--export-py-non-hermetic-scripts-in-resolve=['a', 'b']"]
    )
    rule_runner.set_options(
        [
            *pants_args_for_python_lockfiles,
            f"--python-interpreter-constraints=['=={current_interpreter}']",
            "--python-resolves={'a': 'lock.txt', 'b': 'lock.txt'}",
            "--export-resolve=a",
            "--export-resolve=b",
            "--export-py-editable-in-resolve=['a', 'b']",
            format_flag,
            *hermetic_flags,
        ],
        env_inherit={"PATH", "PYENV_ROOT"},
    )
    all_results = rule_runner.request(ExportResults, [ExportVenvsRequest(targets=())])

    for result, resolve in zip(all_results, ["a", "b"]):
        if py_resolve_format == PythonResolveExportFormat.symlinked_immutable_virtualenv:
            assert len(result.post_processing_cmds) == 2
            ppc0, ppc1 = result.post_processing_cmds
            assert ppc0.argv == ("rmdir", "{digest_root}")
            assert ppc0.extra_env == FrozenDict()
            assert ppc1.argv[0:2] == ("ln", "-s")
            # The third arg is the full path to the venv under the pex_root, which we
            # don't easily know here, so we ignore it in this comparison.
            assert ppc1.argv[3] == "{digest_root}"
            assert ppc1.extra_env == FrozenDict()
        else:
            if resolve == "a":
                # editable wheels are installed for a user resolve that has dists
                assert len(result.post_processing_cmds) == 5
            else:
                # tool resolves (flake8) and user resolves w/o dists (b)
                # do not run the commands to do editable installs
                assert len(result.post_processing_cmds) == 2

            ppc0 = result.post_processing_cmds[0]
            # The first arg is the full path to the python interpreter, which we
            # don't easily know here, so we ignore it in this comparison.

            # The second arg is expected to be tmpdir/./pex.
            tmpdir, pex_pex_name = os.path.split(os.path.normpath(ppc0.argv[1]))
            assert pex_pex_name == "pex"
            assert re.match(r"\{digest_root\}/\.[0-9a-f]{32}\.tmp", tmpdir)

            # The third arg is expected to be tmpdir/{resolve}.pex.
            req_pex_dir, req_pex_name = os.path.split(ppc0.argv[2])
            assert req_pex_dir == tmpdir
            assert req_pex_name == f"{resolve}.pex"

            assert ppc0.argv[3:7] == (
                "venv",
                "--pip",
                "--collisions-ok",
                f"--prompt={resolve}/{current_interpreter}",
            )
            if py_hermetic_scripts:
                assert "--non-hermetic-scripts" not in ppc0.argv
            else:
                assert ppc0.argv[7] == "--non-hermetic-scripts"
            assert ppc0.argv[-1] == "{digest_root}"
            assert ppc0.extra_env["PEX_MODULE"] == "pex.tools"
            assert ppc0.extra_env.get("PEX_ROOT") is not None

            ppc1 = result.post_processing_cmds[-1]
            assert ppc1.argv == ("rm", "-rf", tmpdir)
            assert ppc1.extra_env == FrozenDict()

    reldirs = [result.reldir for result in all_results]
    assert reldirs == [
        f"python/virtualenvs/a/{current_interpreter}",
        f"python/virtualenvs/b/{current_interpreter}",
    ]


def test_export_tool(rule_runner: RuleRunner) -> None:
    """Test exporting an ExportableTool."""
    rule_runner.set_options([*pants_args_for_python_lockfiles, "--export-resolve=isort"])
    results = rule_runner.request(ExportResults, [ExportVenvsRequest(tuple())])
    assert len(results) == 1
    result = results[0]
    assert result.resolve == isort_subsystem.Isort.options_scope
    assert "isort" in result.description


def test_export_codegen_outputs():
    class CodegenSourcesField(SingleSourceField):
        pass

    class CodegenTarget(Target):
        alias = "codegen_target"
        core_fields = (CodegenSourcesField, PythonResolveField)
        help = "n/a"

    class CodegenGenerateSourcesRequest(GenerateSourcesRequest):
        input = CodegenSourcesField
        output = PythonSourceField

    @rule
    async def do_codegen(request: CodegenGenerateSourcesRequest) -> GeneratedSources:
        # Generate a Python file with the same contents as each input file.
        input_files = await get_digest_contents(request.protocol_sources.digest)
        generated_files = [
            dataclasses.replace(input_file, path=input_file.path + ".py")
            for input_file in input_files
        ]
        result = await digest_to_snapshot(
            **implicitly({CreateDigest(generated_files): CreateDigest})
        )
        return GeneratedSources(result)

    rule_runner = RuleRunner(
        rules=[
            *export.rules(),
            *pex_from_targets.rules(),
            *target_types_rules.rules(),
            *distdir.rules(),
            *local_dists_pep660.rules(),
            do_codegen,
            QueryRule(Targets, [RawSpecs]),
            QueryRule(ExportResults, [ExportVenvsRequest]),
            UnionRule(GenerateSourcesRequest, CodegenGenerateSourcesRequest),
        ],
        target_types=[
            PythonRequirementTarget,
            PythonSourcesGeneratorTarget,
            PythonDistribution,
            CodegenTarget,
        ],
    )

    vinfo = sys.version_info
    current_interpreter = f"{vinfo.major}.{vinfo.minor}.{vinfo.micro}"
    rule_runner.set_options(
        [
            *pants_args_for_python_lockfiles,
            f"--python-interpreter-constraints=['=={current_interpreter}']",
            "--python-resolves={'test-resolve': 'test-resolve.lock'}",
            "--source-root-patterns=src/python",
            "--export-resolve=test-resolve",
            "--export-py-generated-sources-in-resolve=['test-resolve']",
        ],
        env_inherit=PYTHON_BOOTSTRAP_ENV,
    )

    rule_runner.write_files(
        {
            "test-resolve.lock": "",
            # Case #1
            # `foo`` package exports `an-input.py` file as a generated source.
            "src/python/foo/BUILD": dedent(
                """
                codegen_target(
                  name="codegen",
                  source="an-input",
                  resolve="test-resolve"
                )
                """
            ),
            "src/python/foo/an-input": "print('Hello World!')\n",
            # Case #2
            # The local `ansicolors`` package exports `ansicolors-input.py` file as a generated source.
            # The local `ansicolors package clashes with the 3rd party dep `ansicolors``.
            # This is an edge case.
            "src/python/ansicolors/BUILD": dedent(
                """
                codegen_target(
                  name="codegen",
                  source="ansicolors-input",
                  resolve="test-resolve"
                )
                """
            ),
            "src/python/ansicolors/ansicolors-input": "print('Hello World!')\n",
        }
    )

    export_results = rule_runner.request(ExportResults, [ExportVenvsRequest(targets=())])
    assert len(export_results) == 1
    export_result = export_results[0]

    export_snapshot = rule_runner.request(Snapshot, [export_result.digest])
    assert any(p.endswith("__pants_codegen__/codegen_setup.py") for p in export_snapshot.files)
    assert any(p.endswith("__pants_codegen__/foo/an-input.py") for p in export_snapshot.files)
    assert any(
        p.endswith("__pants_codegen__/ansicolors/ansicolors-input.py")
        for p in export_snapshot.files
    )

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	import dataclasses	1✔
4	import os	1✔
5	import re	1✔
6	import sys	1✔
7	from textwrap import dedent	1✔
8
9	import pytest	1✔
10
11	from pants.backend.python import target_types_rules	1✔
12	from pants.backend.python.goals import export	1✔
13	from pants.backend.python.goals.export import ExportVenvsRequest, PythonResolveExportFormat	1✔
14	from pants.backend.python.lint.isort import subsystem as isort_subsystem	1✔
15	from pants.backend.python.macros.python_artifact import PythonArtifact	1✔
16	from pants.backend.python.target_types import (	1✔
17	PythonDistribution,
18	PythonRequirementTarget,
19	PythonResolveField,
20	PythonSourceField,
21	PythonSourcesGeneratorTarget,
22	)
23	from pants.backend.python.util_rules import local_dists_pep660, pex_from_targets	1✔
24	from pants.base.specs import RawSpecs	1✔
25	from pants.core.goals.export import ExportResults	1✔
26	from pants.core.util_rules import distdir	1✔
27	from pants.engine.fs import CreateDigest	1✔
28	from pants.engine.internals.native_engine import Snapshot	1✔
29	from pants.engine.internals.parametrize import Parametrize	1✔
30	from pants.engine.intrinsics import digest_to_snapshot, get_digest_contents	1✔
31	from pants.engine.rules import QueryRule, implicitly, rule	1✔
32	from pants.engine.target import (	1✔
33	GeneratedSources,
34	GenerateSourcesRequest,
35	SingleSourceField,
36	Target,
37	Targets,
38	)
39	from pants.engine.unions import UnionRule	1✔
40	from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, RuleRunner	1✔
41	from pants.util.frozendict import FrozenDict	1✔
42
43	pants_args_for_python_lockfiles = [	1✔
44	"--python-enable-resolves=True",
45	# Turn off lockfile validation to make the test simpler.
46	"--python-invalid-lockfile-behavior=ignore",
47	# Turn off python synthetic lockfile targets to make the test simpler.
48	"--no-python-enable-lockfile-targets",
49	]
50
51
52	@pytest.fixture	1✔
53	def rule_runner() -> RuleRunner:	1✔
54	return RuleRunner(	1✔
55	rules=[
56	*export.rules(),
57	*pex_from_targets.rules(),
58	*target_types_rules.rules(),
59	*distdir.rules(),
60	*local_dists_pep660.rules(),
61	*isort_subsystem.rules(), # add a tool that we can try exporting
62	QueryRule(Targets, [RawSpecs]),
63	QueryRule(ExportResults, [ExportVenvsRequest]),
64	],
65	target_types=[PythonRequirementTarget, PythonSourcesGeneratorTarget, PythonDistribution],
66	objects={"python_artifact": PythonArtifact, "parametrize": Parametrize},
67	)
68
69
70	@pytest.mark.parametrize(	1✔
71	"py_resolve_format,py_hermetic_scripts",
72	[
73	(PythonResolveExportFormat.symlinked_immutable_virtualenv, True),
74	(PythonResolveExportFormat.mutable_virtualenv, True),
75	(PythonResolveExportFormat.mutable_virtualenv, False),
76	],
77	)
78	def test_export_venv_new_codepath(	1✔
79	rule_runner: RuleRunner,
80	py_resolve_format: PythonResolveExportFormat,
81	py_hermetic_scripts: bool,
82	) -> None:
83	# We know that the current interpreter exists on the system.
84	vinfo = sys.version_info	1✔
85	current_interpreter = f"{vinfo.major}.{vinfo.minor}.{vinfo.micro}"	1✔
86	rule_runner.write_files(	1✔
87	{
88	"src/foo/__init__.py": "from colors import *",
89	"src/foo/BUILD": dedent(
90	"""\
91	python_sources(name='foo', resolve=parametrize('a', 'b'))
92	python_distribution(
93	name='dist',
94	provides=python_artifact(name='foo', version='1.2.3'),
95	dependencies=[':foo@resolve=a'],
96	)
97	python_requirement(name='req1', requirements=['ansicolors==1.1.8'], resolve='a')
98	python_requirement(name='req2', requirements=['ansicolors==1.1.8'], resolve='b')
99	"""
100	),
101	"lock.txt": "ansicolors==1.1.8",
102	}
103	)
104
105	format_flag = f"--export-py-resolve-format={py_resolve_format.value}"	1✔
106	hermetic_flags = (	1✔
107	[] if py_hermetic_scripts else ["--export-py-non-hermetic-scripts-in-resolve=['a', 'b']"]
108	)
109	rule_runner.set_options(	1✔
110	[
111	*pants_args_for_python_lockfiles,
112	f"--python-interpreter-constraints=['=={current_interpreter}']",
113	"--python-resolves={'a': 'lock.txt', 'b': 'lock.txt'}",
114	"--export-resolve=a",
115	"--export-resolve=b",
116	"--export-py-editable-in-resolve=['a', 'b']",
117	format_flag,
118	*hermetic_flags,
119	],
120	env_inherit={"PATH", "PYENV_ROOT"},
121	)
122	all_results = rule_runner.request(ExportResults, [ExportVenvsRequest(targets=())])	1✔
123
124	for result, resolve in zip(all_results, ["a", "b"]):	1✔
125	if py_resolve_format == PythonResolveExportFormat.symlinked_immutable_virtualenv:	1✔
126	assert len(result.post_processing_cmds) == 2	1✔
127	ppc0, ppc1 = result.post_processing_cmds	1✔
128	assert ppc0.argv == ("rmdir", "{digest_root}")	1✔
129	assert ppc0.extra_env == FrozenDict()	1✔
130	assert ppc1.argv[0:2] == ("ln", "-s")	1✔
131	# The third arg is the full path to the venv under the pex_root, which we
132	# don't easily know here, so we ignore it in this comparison.
133	assert ppc1.argv[3] == "{digest_root}"	1✔
134	assert ppc1.extra_env == FrozenDict()	1✔
135	else:
136	if resolve == "a":	1✔
137	# editable wheels are installed for a user resolve that has dists
138	assert len(result.post_processing_cmds) == 5	1✔
139	else:
140	# tool resolves (flake8) and user resolves w/o dists (b)
141	# do not run the commands to do editable installs
142	assert len(result.post_processing_cmds) == 2	1✔
143
144	ppc0 = result.post_processing_cmds[0]	1✔
145	# The first arg is the full path to the python interpreter, which we
146	# don't easily know here, so we ignore it in this comparison.
147
148	# The second arg is expected to be tmpdir/./pex.
149	tmpdir, pex_pex_name = os.path.split(os.path.normpath(ppc0.argv[1]))	1✔
150	assert pex_pex_name == "pex"	1✔
151	assert re.match(r"\{digest_root\}/\.[0-9a-f]{32}\.tmp", tmpdir)	1✔
152
153	# The third arg is expected to be tmpdir/{resolve}.pex.
154	req_pex_dir, req_pex_name = os.path.split(ppc0.argv[2])	1✔
155	assert req_pex_dir == tmpdir	1✔
156	assert req_pex_name == f"{resolve}.pex"	1✔
157
158	assert ppc0.argv[3:7] == (	1✔
159	"venv",
160	"--pip",
161	"--collisions-ok",
162	f"--prompt={resolve}/{current_interpreter}",
163	)
164	if py_hermetic_scripts:	1✔
165	assert "--non-hermetic-scripts" not in ppc0.argv	1✔
166	else:
167	assert ppc0.argv[7] == "--non-hermetic-scripts"	1✔
168	assert ppc0.argv[-1] == "{digest_root}"	1✔
169	assert ppc0.extra_env["PEX_MODULE"] == "pex.tools"	1✔
170	assert ppc0.extra_env.get("PEX_ROOT") is not None	1✔
171
172	ppc1 = result.post_processing_cmds[-1]	1✔
173	assert ppc1.argv == ("rm", "-rf", tmpdir)	1✔
174	assert ppc1.extra_env == FrozenDict()	1✔
175
176	reldirs = [result.reldir for result in all_results]	1✔
177	assert reldirs == [	1✔
178	f"python/virtualenvs/a/{current_interpreter}",
179	f"python/virtualenvs/b/{current_interpreter}",
180	]
181
182
183	def test_export_tool(rule_runner: RuleRunner) -> None:	1✔
184	"""Test exporting an ExportableTool."""
185	rule_runner.set_options([*pants_args_for_python_lockfiles, "--export-resolve=isort"])	1✔
186	results = rule_runner.request(ExportResults, [ExportVenvsRequest(tuple())])	1✔
UNCOV 187	assert len(results) == 1	×
UNCOV 188	result = results[0]	×
UNCOV 189	assert result.resolve == isort_subsystem.Isort.options_scope	×
UNCOV 190	assert "isort" in result.description	×
191
192
193	def test_export_codegen_outputs():	1✔
194	class CodegenSourcesField(SingleSourceField):	1✔
195	pass	1✔
196
197	class CodegenTarget(Target):	1✔
198	alias = "codegen_target"	1✔
199	core_fields = (CodegenSourcesField, PythonResolveField)	1✔
200	help = "n/a"	1✔
201
202	class CodegenGenerateSourcesRequest(GenerateSourcesRequest):	1✔
203	input = CodegenSourcesField	1✔
204	output = PythonSourceField	1✔
205
206	@rule	1✔
207	async def do_codegen(request: CodegenGenerateSourcesRequest) -> GeneratedSources:	1✔
208	# Generate a Python file with the same contents as each input file.
209	input_files = await get_digest_contents(request.protocol_sources.digest)	1✔
210	generated_files = [	1✔
211	dataclasses.replace(input_file, path=input_file.path + ".py")
212	for input_file in input_files
213	]
214	result = await digest_to_snapshot(	1✔
215	**implicitly({CreateDigest(generated_files): CreateDigest})
216	)
217	return GeneratedSources(result)	1✔
218
219	rule_runner = RuleRunner(	1✔
220	rules=[
221	*export.rules(),
222	*pex_from_targets.rules(),
223	*target_types_rules.rules(),
224	*distdir.rules(),
225	*local_dists_pep660.rules(),
226	do_codegen,
227	QueryRule(Targets, [RawSpecs]),
228	QueryRule(ExportResults, [ExportVenvsRequest]),
229	UnionRule(GenerateSourcesRequest, CodegenGenerateSourcesRequest),
230	],
231	target_types=[
232	PythonRequirementTarget,
233	PythonSourcesGeneratorTarget,
234	PythonDistribution,
235	CodegenTarget,
236	],
237	)
238
239	vinfo = sys.version_info	1✔
240	current_interpreter = f"{vinfo.major}.{vinfo.minor}.{vinfo.micro}"	1✔
241	rule_runner.set_options(	1✔
242	[
243	*pants_args_for_python_lockfiles,
244	f"--python-interpreter-constraints=['=={current_interpreter}']",
245	"--python-resolves={'test-resolve': 'test-resolve.lock'}",
246	"--source-root-patterns=src/python",
247	"--export-resolve=test-resolve",
248	"--export-py-generated-sources-in-resolve=['test-resolve']",
249	],
250	env_inherit=PYTHON_BOOTSTRAP_ENV,
251	)
252
253	rule_runner.write_files(	1✔
254	{
255	"test-resolve.lock": "",
256	# Case #1
257	# `foo`` package exports `an-input.py` file as a generated source.
258	"src/python/foo/BUILD": dedent(
259	"""
260	codegen_target(
261	name="codegen",
262	source="an-input",
263	resolve="test-resolve"
264	)
265	"""
266	),
267	"src/python/foo/an-input": "print('Hello World!')\n",
268	# Case #2
269	# The local `ansicolors`` package exports `ansicolors-input.py` file as a generated source.
270	# The local `ansicolors package clashes with the 3rd party dep `ansicolors``.
271	# This is an edge case.
272	"src/python/ansicolors/BUILD": dedent(
273	"""
274	codegen_target(
275	name="codegen",
276	source="ansicolors-input",
277	resolve="test-resolve"
278	)
279	"""
280	),
281	"src/python/ansicolors/ansicolors-input": "print('Hello World!')\n",
282	}
283	)
284
285	export_results = rule_runner.request(ExportResults, [ExportVenvsRequest(targets=())])	1✔
286	assert len(export_results) == 1	1✔
287	export_result = export_results[0]	1✔
288
289	export_snapshot = rule_runner.request(Snapshot, [export_result.digest])	1✔
290	assert any(p.endswith("__pants_codegen__/codegen_setup.py") for p in export_snapshot.files)	1✔
291	assert any(p.endswith("__pants_codegen__/foo/an-input.py") for p in export_snapshot.files)	1✔
292	assert any(	1✔
293	p.endswith("__pants_codegen__/ansicolors/ansicolors-input.py")
294	for p in export_snapshot.files
295	)

pantsbuild / pants / 18847018991

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous