22285099215

Committed 22 Feb 2026 08:52PM UTC coverage: 75.854% (-17.1%) from 92.936%

Build # 22285099215

Build Type

Pull #23121

github

Committed by

web-flow

Commit Message

Merge c7299df9c into ba8359840

Pull Request Pull Request #23121: fix issue with optional fields in dependency validator

Run Details

28 of 29 new or added lines in 2 files covered. (96.55%)

11174 existing lines in 400 files now uncovered.

53694 of 70786 relevant lines covered (75.85%)

1.88 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.26

/src/python/pants/core/util_rules/adhoc_binaries.py

# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import os
import sys
from dataclasses import dataclass
from textwrap import dedent  # noqa: PNT20

from pants.core.environments.target_types import EnvironmentTarget
from pants.core.subsystems.python_bootstrap import PythonBootstrapSubsystem
from pants.core.util_rules.system_binaries import BashBinary, SystemBinariesSubsystem, TarBinary
from pants.engine.download_file import download_file
from pants.engine.fs import DownloadFile
from pants.engine.internals.native_engine import FileDigest
from pants.engine.platform import Platform
from pants.engine.process import Process, ProcessCacheScope, execute_process_or_raise
from pants.engine.rules import collect_rules, implicitly, rule
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel


@dataclass(frozen=True)
class PythonBuildStandaloneBinary:
    """A Python interpreter for use by `@rule` code as an alternative to BashBinary scripts.

    This interpreter is provided by Python Build Standalone https://gregoryszorc.com/docs/python-build-standalone/main/,
    which has a few caveats. Namely it doesn't play nicely with third-party sdists. Meaning Pants'
    scripts being run by Python Build Standalone should avoid third-party sdists.
    """

    _CACHE_DIRNAME = "python_build_standalone"
    _SYMLINK_DIRNAME = ".python-build-standalone"
    APPEND_ONLY_CACHES = FrozenDict({_CACHE_DIRNAME: _SYMLINK_DIRNAME})

    path: str  # The absolute path to a Python executable


# NB: These private types are solely so we can test the docker-path using the local
# environment.
class _PythonBuildStandaloneBinary(PythonBuildStandaloneBinary):
    pass


class _DownloadPythonBuildStandaloneBinaryRequest:
    pass


@rule(desc="Downloading Python for scripts", level=LogLevel.TRACE)
async def download_python_binary(
    _: _DownloadPythonBuildStandaloneBinaryRequest,
    platform: Platform,
    tar_binary: TarBinary,
    bash_binary: BashBinary,
    python_bootstrap: PythonBootstrapSubsystem,
    system_binaries_environment: SystemBinariesSubsystem.EnvironmentAware,
) -> _PythonBuildStandaloneBinary:
    url, fingerprint, bytelen = python_bootstrap.internal_python_build_standalone_info[
        platform.value
    ]

    filename = url.rsplit("/", 1)[-1]
    python_archive = await download_file(
        DownloadFile(
            url,
            FileDigest(
                fingerprint=fingerprint,
                serialized_bytes_length=bytelen,
            ),
        ),
        **implicitly(),
    )

    download_result = await execute_process_or_raise(
        **implicitly(
            Process(
                argv=[tar_binary.path, "-xvf", filename],
                input_digest=python_archive,
                env={"PATH": os.pathsep.join(system_binaries_environment.system_binary_paths)},
                description="Extract Pants' execution Python",
                level=LogLevel.DEBUG,
                output_directories=("python",),
            )
        )
    )

    installation_root = f"{PythonBuildStandaloneBinary._SYMLINK_DIRNAME}/{download_result.output_digest.fingerprint}"

    # NB: This is similar to what we do for every Python provider. We should refactor these into
    # some shared code to centralize the behavior.
    installation_script = dedent(
        f"""\
        if [ ! -f "{installation_root}/DONE" ]; then
            cp -r python "{installation_root}"
            touch "{installation_root}/DONE"
        fi
        echo "$(realpath "{installation_root}")/bin/python3"
    """
    )

    result = await execute_process_or_raise(
        **implicitly(
            Process(
                [bash_binary.path, "-c", installation_script],
                level=LogLevel.DEBUG,
                input_digest=download_result.output_digest,
                description="Install Python for Pants usage",
                env={"PATH": os.pathsep.join(system_binaries_environment.system_binary_paths)},
                append_only_caches=PythonBuildStandaloneBinary.APPEND_ONLY_CACHES,
                # Don't cache, we want this to always be run so that we can assume for the rest of the
                # session the named_cache destination for this Python is valid, as the Python ecosystem
                # mainly assumes absolute paths for Python interpreters.
                cache_scope=ProcessCacheScope.PER_SESSION,
            )
        )
    )

    return _PythonBuildStandaloneBinary(result.stdout.decode().splitlines()[-1].strip())


@rule
async def get_python_for_scripts(env_tgt: EnvironmentTarget) -> PythonBuildStandaloneBinary:
    if env_tgt.can_access_local_system_paths:
        return PythonBuildStandaloneBinary(sys.executable)

    result = await download_python_binary(
        _DownloadPythonBuildStandaloneBinaryRequest(), **implicitly()
    )

    return PythonBuildStandaloneBinary(result.path)


@dataclass(frozen=True)
class GunzipBinaryRequest:
    pass


@dataclass(frozen=True)
class GunzipBinary:
    python_binary: PythonBuildStandaloneBinary

    def extract_archive_argv(self, archive_path: str, extract_path: str) -> tuple[str, ...]:
        archive_name = os.path.basename(archive_path)
        dest_file_name = os.path.splitext(archive_name)[0]
        dest_path = os.path.join(extract_path, dest_file_name)
        script = dedent(
            f"""
            import gzip
            import shutil
            with gzip.GzipFile(filename={archive_path!r}, mode="rb") as source:
                with open({dest_path!r}, "wb") as dest:
                    shutil.copyfileobj(source, dest)
            """
        )
        return (self.python_binary.path, "-c", script)


@rule
async def find_gunzip(python_binary: PythonBuildStandaloneBinary) -> GunzipBinary:
    return GunzipBinary(python_binary)


@rule
async def find_gunzip_wrapper(_: GunzipBinaryRequest, gunzip: GunzipBinary) -> GunzipBinary:
    return gunzip


def rules():
    return collect_rules()

1	# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	4✔
5
6	import os	4✔
7	import sys	4✔
8	from dataclasses import dataclass	4✔
9	from textwrap import dedent # noqa: PNT20	4✔
10
11	from pants.core.environments.target_types import EnvironmentTarget	4✔
12	from pants.core.subsystems.python_bootstrap import PythonBootstrapSubsystem	4✔
13	from pants.core.util_rules.system_binaries import BashBinary, SystemBinariesSubsystem, TarBinary	4✔
14	from pants.engine.download_file import download_file	4✔
15	from pants.engine.fs import DownloadFile	4✔
16	from pants.engine.internals.native_engine import FileDigest	4✔
17	from pants.engine.platform import Platform	4✔
18	from pants.engine.process import Process, ProcessCacheScope, execute_process_or_raise	4✔
19	from pants.engine.rules import collect_rules, implicitly, rule	4✔
20	from pants.util.frozendict import FrozenDict	4✔
21	from pants.util.logging import LogLevel	4✔
22
23
24	@dataclass(frozen=True)	4✔
25	class PythonBuildStandaloneBinary:	4✔
26	"""A Python interpreter for use by `@rule` code as an alternative to BashBinary scripts.
27
28	This interpreter is provided by Python Build Standalone https://gregoryszorc.com/docs/python-build-standalone/main/,
29	which has a few caveats. Namely it doesn't play nicely with third-party sdists. Meaning Pants'
30	scripts being run by Python Build Standalone should avoid third-party sdists.
31	"""
32
33	_CACHE_DIRNAME = "python_build_standalone"	4✔
34	_SYMLINK_DIRNAME = ".python-build-standalone"	4✔
35	APPEND_ONLY_CACHES = FrozenDict({_CACHE_DIRNAME: _SYMLINK_DIRNAME})	4✔
36
37	path: str # The absolute path to a Python executable	4✔
38
39
40	# NB: These private types are solely so we can test the docker-path using the local
41	# environment.
42	class _PythonBuildStandaloneBinary(PythonBuildStandaloneBinary):	4✔
43	pass	4✔
44
45
46	class _DownloadPythonBuildStandaloneBinaryRequest:	4✔
47	pass	4✔
48
49
50	@rule(desc="Downloading Python for scripts", level=LogLevel.TRACE)	4✔
51	async def download_python_binary(	4✔
52	_: _DownloadPythonBuildStandaloneBinaryRequest,
53	platform: Platform,
54	tar_binary: TarBinary,
55	bash_binary: BashBinary,
56	python_bootstrap: PythonBootstrapSubsystem,
57	system_binaries_environment: SystemBinariesSubsystem.EnvironmentAware,
58	) -> _PythonBuildStandaloneBinary:
UNCOV 59	url, fingerprint, bytelen = python_bootstrap.internal_python_build_standalone_info[	×
60	platform.value
61	]
62
UNCOV 63	filename = url.rsplit("/", 1)[-1]	×
UNCOV 64	python_archive = await download_file(	×
65	DownloadFile(
66	url,
67	FileDigest(
68	fingerprint=fingerprint,
69	serialized_bytes_length=bytelen,
70	),
71	),
72	**implicitly(),
73	)
74
UNCOV 75	download_result = await execute_process_or_raise(	×
76	**implicitly(
77	Process(
78	argv=[tar_binary.path, "-xvf", filename],
79	input_digest=python_archive,
80	env={"PATH": os.pathsep.join(system_binaries_environment.system_binary_paths)},
81	description="Extract Pants' execution Python",
82	level=LogLevel.DEBUG,
83	output_directories=("python",),
84	)
85	)
86	)
87
UNCOV 88	installation_root = f"{PythonBuildStandaloneBinary._SYMLINK_DIRNAME}/{download_result.output_digest.fingerprint}"	×
89
90	# NB: This is similar to what we do for every Python provider. We should refactor these into
91	# some shared code to centralize the behavior.
UNCOV 92	installation_script = dedent(	×
93	f"""\
94	if [ ! -f "{installation_root}/DONE" ]; then
95	cp -r python "{installation_root}"
96	touch "{installation_root}/DONE"
97	fi
98	echo "$(realpath "{installation_root}")/bin/python3"
99	"""
100	)
101
UNCOV 102	result = await execute_process_or_raise(	×
103	**implicitly(
104	Process(
105	[bash_binary.path, "-c", installation_script],
106	level=LogLevel.DEBUG,
107	input_digest=download_result.output_digest,
108	description="Install Python for Pants usage",
109	env={"PATH": os.pathsep.join(system_binaries_environment.system_binary_paths)},
110	append_only_caches=PythonBuildStandaloneBinary.APPEND_ONLY_CACHES,
111	# Don't cache, we want this to always be run so that we can assume for the rest of the
112	# session the named_cache destination for this Python is valid, as the Python ecosystem
113	# mainly assumes absolute paths for Python interpreters.
114	cache_scope=ProcessCacheScope.PER_SESSION,
115	)
116	)
117	)
118
UNCOV 119	return _PythonBuildStandaloneBinary(result.stdout.decode().splitlines()[-1].strip())	×
120
121
122	@rule	4✔
123	async def get_python_for_scripts(env_tgt: EnvironmentTarget) -> PythonBuildStandaloneBinary:	4✔
124	if env_tgt.can_access_local_system_paths:	4✔
125	return PythonBuildStandaloneBinary(sys.executable)	4✔
126
UNCOV 127	result = await download_python_binary(	×
128	_DownloadPythonBuildStandaloneBinaryRequest(), **implicitly()
129	)
130
UNCOV 131	return PythonBuildStandaloneBinary(result.path)	×
132
133
134	@dataclass(frozen=True)	4✔
135	class GunzipBinaryRequest:	4✔
136	pass	4✔
137
138
139	@dataclass(frozen=True)	4✔
140	class GunzipBinary:	4✔
141	python_binary: PythonBuildStandaloneBinary	4✔
142
143	def extract_archive_argv(self, archive_path: str, extract_path: str) -> tuple[str, ...]:	4✔
144	archive_name = os.path.basename(archive_path)	4✔
145	dest_file_name = os.path.splitext(archive_name)[0]	4✔
146	dest_path = os.path.join(extract_path, dest_file_name)	4✔
147	script = dedent(	4✔
148	f"""
149	import gzip
150	import shutil
151	with gzip.GzipFile(filename={archive_path!r}, mode="rb") as source:
152	with open({dest_path!r}, "wb") as dest:
153	shutil.copyfileobj(source, dest)
154	"""
155	)
156	return (self.python_binary.path, "-c", script)	4✔
157
158
159	@rule	4✔
160	async def find_gunzip(python_binary: PythonBuildStandaloneBinary) -> GunzipBinary:	4✔
161	return GunzipBinary(python_binary)	4✔
162
163
164	@rule	4✔
165	async def find_gunzip_wrapper(_: GunzipBinaryRequest, gunzip: GunzipBinary) -> GunzipBinary:	4✔
166	return gunzip	×
167
168
169	def rules():	4✔
170	return collect_rules()	4✔

pantsbuild / pants / 22285099215

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous