18252174847

Committed 05 Oct 2025 01:36AM UTC coverage: 43.382% (-36.9%) from 80.261%

Build # 18252174847

Build Type

push

github

Committed by

web-flow

Commit Message

run tests on mac arm (#22717)

Just doing the minimal to pull forward the x86_64 pattern.

ref #20993

Run Details

25776 of 59416 relevant lines covered (43.38%)

1.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/python/pants/backend/python/util_rules/local_dists_pep660.py

# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import json
import logging
import os
import shlex
from collections import defaultdict
from dataclasses import dataclass

from pants.backend.python.subsystems.setup import PythonSetup
from pants.backend.python.target_types import PythonProvidesField, PythonResolveField
from pants.backend.python.util_rules import package_dists
from pants.backend.python.util_rules.dists import BuildBackendError, DistBuildRequest
from pants.backend.python.util_rules.dists import rules as dists_rules
from pants.backend.python.util_rules.package_dists import (
    DependencyOwner,
    ExportedTarget,
    create_dist_build_request,
    get_owned_dependencies,
)
from pants.backend.python.util_rules.pex import PexRequest, VenvPexProcess, create_venv_pex
from pants.base.build_root import BuildRoot
from pants.core.util_rules import system_binaries
from pants.core.util_rules.system_binaries import BashBinary, UnzipBinary
from pants.engine.fs import (
    CreateDigest,
    Digest,
    DigestSubset,
    FileContent,
    MergeDigests,
    PathGlobs,
    RemovePrefix,
)
from pants.engine.internals.graph import resolve_target
from pants.engine.internals.native_engine import Address
from pants.engine.intrinsics import create_digest, digest_to_snapshot, merge_digests, remove_prefix
from pants.engine.process import Process, fallible_to_exec_result_or_raise
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
from pants.engine.target import AllTargets, Target, Targets, WrappedTargetRequest
from pants.engine.unions import UnionMembership
from pants.util.docutil import doc_url
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel
from pants.util.osutil import is_macos_big_sur
from pants.util.resources import read_resource
from pants.util.strutil import softwrap

logger = logging.getLogger(__name__)


_scripts_package = "pants.backend.python.util_rules.scripts"


@dataclass(frozen=True)
class PEP660BuildResult:
    output: Digest
    # Relpaths in the output digest.
    editable_wheel_path: str | None


def dump_backend_wrapper_json(
    dist_dir: str,
    pth_file_path: str,
    direct_url: str,
    request: DistBuildRequest,
) -> bytes:
    """Build the settings json for our PEP 517 / PEP 660 wrapper script."""

    def clean_config_settings(
        cs: FrozenDict[str, tuple[str, ...]] | None,
    ) -> dict[str, list[str]] | None:
        # setuptools.build_meta expects list values and chokes on tuples.
        # We assume/hope that other backends accept lists as well.
        return None if cs is None else {k: list(v) for k, v in cs.items()}

    # tag the editable wheel as widely compatible
    lang_tag, abi_tag, platform_tag = "py3", "none", "any"
    if request.interpreter_constraints.includes_python2():
        # Assume everything has py3 support. If not, we'll need a new includes_python3 method.
        lang_tag = "py2.py3"

    settings = {
        "build_backend": request.build_system.build_backend,
        "dist_dir": dist_dir,
        "pth_file_path": pth_file_path,
        "wheel_config_settings": clean_config_settings(request.wheel_config_settings),
        "tags": "-".join([lang_tag, abi_tag, platform_tag]),
        "direct_url": direct_url,
    }
    return json.dumps(settings).encode()


@rule
async def run_pep660_build(
    request: DistBuildRequest, python_setup: PythonSetup, build_root: BuildRoot
) -> PEP660BuildResult:
    """Run our PEP 517 / PEP 660 wrapper script to generate an editable wheel.

    The PEP 517 / PEP 660 wrapper script is responsible for building the editable wheel.
    The backend wrapper script, along with the commands that install the editable wheel,
    need to conform to the following specs so that Pants is a PEP 660 compliant frontend,
    a PEP 660 compliant backend, and that it builds a compliant wheel and install.

    NOTE: PEP 660 does not address the `.data` directory, so the wrapper ignores it.

    Relevant Specs:
      https://peps.python.org/pep-0517/
      https://peps.python.org/pep-0660/
      https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/
      https://packaging.python.org/en/latest/specifications/recording-installed-packages/
      https://packaging.python.org/en/latest/specifications/direct-url-data-structure/
      https://packaging.python.org/en/latest/specifications/binary-distribution-format/
    """
    dist_abs_path = (
        build_root.path
        if request.dist_source_root == "."
        else str(build_root.pathlib_path / request.dist_source_root)
    )
    direct_url = "file://" + dist_abs_path.replace(os.path.sep, "/")

    # Create the .pth files to add the relevant source root to sys.path.
    # We cannot use the build backend to do this because we do not want to tell
    # it where the workspace is and risk it adding anything there.
    # NOTE: We use .pth files to support ICs less than python3.7.
    #       A future enhancement might be to provide more precise editable
    #       wheels based on https://pypi.org/project/editables/, but that only
    #       supports python3.7+ (what pip supports as of April 2023).
    #       Or maybe do something like setuptools strict editable wheel.
    pth_file_contents = ""
    for source_root in request.build_time_source_roots:  # NB: the roots are sorted
        # Can we use just the dist_abs_path instead of including all source roots?
        abs_path = (
            build_root.path if source_root == "." else str(build_root.pathlib_path / source_root)
        )
        pth_file_contents += f"{abs_path}\n"
    pth_file_name = "__pants__.pth"
    pth_file_path = os.path.join(request.working_directory, pth_file_name)

    # This is the setuptools dist directory, not Pants's, so we hardcode to dist/.
    dist_dir = "dist"
    dist_output_dir = os.path.join(dist_dir, request.output_path)

    backend_wrapper_json = "backend_wrapper.json"
    backend_wrapper_json_path = os.path.join(request.working_directory, backend_wrapper_json)
    backend_wrapper_name = "backend_wrapper.py"
    backend_wrapper_path = os.path.join(request.working_directory, backend_wrapper_name)
    backend_wrapper_content = read_resource(_scripts_package, "pep660_backend_wrapper.py")
    assert backend_wrapper_content is not None

    conf_digest, backend_wrapper_digest, build_backend_pex = await concurrently(
        create_digest(
            CreateDigest(
                [
                    FileContent(pth_file_path, pth_file_contents.encode()),
                    FileContent(
                        backend_wrapper_json_path,
                        dump_backend_wrapper_json(
                            dist_output_dir, pth_file_name, direct_url, request
                        ),
                    ),
                ]
            )
        ),
        # The backend_wrapper has its own digest for cache reuse.
        create_digest(CreateDigest([FileContent(backend_wrapper_path, backend_wrapper_content)])),
        # Note that this pex has no entrypoint. We use it to run our wrapper, which
        # in turn imports from and invokes the build backend.
        create_venv_pex(
            **implicitly(
                PexRequest(
                    output_filename="build_backend.pex",
                    internal_only=True,
                    requirements=request.build_system.requires,
                    pex_path=request.extra_build_time_requirements,
                    interpreter_constraints=request.interpreter_constraints,
                )
            )
        ),
    )

    merged_digest = await merge_digests(
        MergeDigests((request.input, conf_digest, backend_wrapper_digest))
    )

    extra_env = {
        **(request.extra_build_time_env or {}),
        "PEX_EXTRA_SYS_PATH": os.pathsep.join(request.build_time_source_roots),
    }
    if python_setup.macos_big_sur_compatibility and is_macos_big_sur():
        extra_env["MACOSX_DEPLOYMENT_TARGET"] = "10.16"

    result = await fallible_to_exec_result_or_raise(
        **implicitly(
            VenvPexProcess(
                build_backend_pex,
                argv=(backend_wrapper_name, backend_wrapper_json),
                input_digest=merged_digest,
                extra_env=extra_env,
                working_directory=request.working_directory,
                output_directories=(dist_dir,),  # Relative to the working_directory.
                description=(
                    f"Run {request.build_system.build_backend} to gather .dist-info for {request.target_address_spec}"
                    if request.target_address_spec
                    else f"Run {request.build_system.build_backend} to gather .dist-info"
                ),
                level=LogLevel.DEBUG,
            )
        )
    )
    output_lines = result.stdout.decode().splitlines()
    line_prefix = "editable_path: "
    editable_path = ""
    for line in output_lines:
        if line.startswith(line_prefix):
            editable_path = os.path.join(request.output_path, line[len(line_prefix) :].strip())
            break

    # Note that output_digest paths are relative to the working_directory.
    output_digest = await remove_prefix(RemovePrefix(result.output_digest, dist_dir))
    output_snapshot = await digest_to_snapshot(output_digest)
    if editable_path not in output_snapshot.files:
        raise BuildBackendError(
            softwrap(
                f"""
                Failed to build PEP 660 editable wheel {editable_path}
                (extracted dist-info from PEP 517 build backend
                {request.build_system.build_backend}).
                """
            )
        )
    return PEP660BuildResult(output_digest, editable_wheel_path=editable_path)


@dataclass(frozen=True)
class LocalDistPEP660Wheels:
    """Contains the PEP 660 "editable" wheels isolated from a single local Python distribution."""

    pep660_wheel_paths: tuple[str, ...]
    pep660_wheels_digest: Digest
    provided_files: frozenset[str]


@rule
async def isolate_local_dist_pep660_wheels(
    dist_target_address: Address,
    bash: BashBinary,
    unzip_binary: UnzipBinary,
    python_setup: PythonSetup,
    union_membership: UnionMembership,
) -> LocalDistPEP660Wheels:
    dist_build_request = await create_dist_build_request(
        dist_target_address=dist_target_address,
        python_setup=python_setup,
        union_membership=union_membership,
        # editable wheel ignores build_wheel+build_sdist args
        validate_wheel_sdist=False,
    )
    pep660_result = await run_pep660_build(dist_build_request, **implicitly())

    # the output digest should only contain wheels, but filter to be safe.
    wheels_snapshot = await digest_to_snapshot(
        **implicitly(DigestSubset(pep660_result.output, PathGlobs(["**/*.whl"])))
    )

    wheels = tuple(sorted(wheels_snapshot.files))

    if not wheels:
        tgt = await resolve_target(
            WrappedTargetRequest(dist_target_address, description_of_origin="<infallible>"),
            **implicitly(),
        )
        logger.warning(
            softwrap(
                f"""
                Encountered a dependency on the {tgt.target.alias} target at {dist_target_address},
                but this target does not produce a Python wheel artifact. Therefore this target's
                code will be used directly from sources, without a distribution being built,
                and any native extensions in it will not be built.

                See {doc_url("docs/python/overview/building-distributions")} for details on how to set up a
                {tgt.target.alias} target to produce a wheel.
                """
            )
        )

    wheels_listing_result = await fallible_to_exec_result_or_raise(
        **implicitly(
            Process(
                argv=[
                    bash.path,
                    "-c",
                    f"""
                set -ex
                for f in {" ".join(shlex.quote(f) for f in wheels)}; do
                  {unzip_binary.path} -Z1 "$f"
                done
                """,
                ],
                input_digest=wheels_snapshot.digest,
                description=f"List contents of editable artifacts produced by {dist_target_address}",
            )
        )
    )
    provided_files = set(wheels_listing_result.stdout.decode().splitlines())

    return LocalDistPEP660Wheels(wheels, wheels_snapshot.digest, frozenset(sorted(provided_files)))


@dataclass(frozen=True)
class AllPythonDistributionTargets:
    targets: Targets


@rule(desc="Find all Python Distribution targets in project", level=LogLevel.DEBUG)
async def find_all_python_distributions(
    all_targets: AllTargets,
) -> AllPythonDistributionTargets:
    return AllPythonDistributionTargets(
        # 'provides' is the field used in PythonDistributionFieldSet
        Targets(tgt for tgt in all_targets if tgt.has_field(PythonProvidesField))
    )


@dataclass(frozen=True)
class ResolveSortedPythonDistributionTargets:
    targets: FrozenDict[str | None, tuple[Target, ...]]


@rule(
    desc="Associate resolves with all Python Distribution targets in project", level=LogLevel.DEBUG
)
async def sort_all_python_distributions_by_resolve(
    all_dists: AllPythonDistributionTargets,
    python_setup: PythonSetup,
) -> ResolveSortedPythonDistributionTargets:
    dists = defaultdict(list)

    if not python_setup.enable_resolves:
        resolve = None
        return ResolveSortedPythonDistributionTargets(
            FrozenDict({resolve: tuple(all_dists.targets)})
        )

    dist_owned_deps = await concurrently(
        get_owned_dependencies(DependencyOwner(ExportedTarget(tgt)), **implicitly())
        for tgt in all_dists.targets
    )

    for dist, owned_deps in zip(all_dists.targets, dist_owned_deps):
        resolve = None
        # assumption: all owned deps are in the same resolve
        for dep in owned_deps:
            if dep.target.has_field(PythonResolveField):
                resolve = dep.target[PythonResolveField].normalized_value(python_setup)
                break
        dists[resolve].append(dist)
    return ResolveSortedPythonDistributionTargets(
        FrozenDict({resolve: tuple(sorted(targets)) for resolve, targets in dists.items()})
    )


@dataclass(frozen=True)
class EditableLocalDistsRequest:
    """Request to generate PEP660 wheels of local dists in the given resolve.

    The editable wheel files must not be exported or made available to the end-user (according to
    PEP 660). Instead, the PEP660 editable wheels serve as intermediate, internal-only,
    representation of what should be installed in the exported virtualenv to create the editable
    installs of local python_distributions.
    """

    resolve: str | None  # None if resolves is not enabled


@dataclass(frozen=True)
class EditableLocalDists:
    """A Digest populated by editable (PEP660) wheels of local dists.

    According to PEP660, these wheels should not be exported to users and must be discarded
    after install. Anything that uses this should ensure that these wheels get installed and
    then deleted.

    Installing PEP660 wheels creates an "editable" install such that the sys.path gets
    adjusted to include source directories from the build root (not from the sandbox).
    This is decidedly not hermetic or portable and should only be used locally.

    PEP660 wheels have .dist-info metadata and the .pth files (or similar) that adjust sys.path.
    """

    optional_digest: Digest | None


@rule(desc="Building editable local distributions (PEP 660)")
async def build_editable_local_dists(
    request: EditableLocalDistsRequest,
    all_dists: ResolveSortedPythonDistributionTargets,
    python_setup: PythonSetup,
) -> EditableLocalDists:
    resolve = request.resolve if python_setup.enable_resolves else None
    resolve_dists = all_dists.targets.get(resolve, ())

    if not resolve_dists:
        return EditableLocalDists(None)

    local_dists_wheels = await concurrently(
        isolate_local_dist_pep660_wheels(target.address, **implicitly()) for target in resolve_dists
    )

    wheels: list[str] = []
    wheels_digests = []
    for local_dist_wheels in local_dists_wheels:
        wheels.extend(local_dist_wheels.pep660_wheel_paths)
        wheels_digests.append(local_dist_wheels.pep660_wheels_digest)

    wheels_digest = await merge_digests(MergeDigests(wheels_digests))

    return EditableLocalDists(wheels_digest)


def rules():
    return (
        *collect_rules(),
        *dists_rules(),
        *package_dists.rules(),
        *system_binaries.rules(),
    )

1	# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	×
5
6	import json	×
7	import logging	×
8	import os	×
9	import shlex	×
10	from collections import defaultdict	×
11	from dataclasses import dataclass	×
12
13	from pants.backend.python.subsystems.setup import PythonSetup	×
14	from pants.backend.python.target_types import PythonProvidesField, PythonResolveField	×
15	from pants.backend.python.util_rules import package_dists	×
16	from pants.backend.python.util_rules.dists import BuildBackendError, DistBuildRequest	×
17	from pants.backend.python.util_rules.dists import rules as dists_rules	×
18	from pants.backend.python.util_rules.package_dists import (	×
19	DependencyOwner,
20	ExportedTarget,
21	create_dist_build_request,
22	get_owned_dependencies,
23	)
24	from pants.backend.python.util_rules.pex import PexRequest, VenvPexProcess, create_venv_pex	×
25	from pants.base.build_root import BuildRoot	×
26	from pants.core.util_rules import system_binaries	×
27	from pants.core.util_rules.system_binaries import BashBinary, UnzipBinary	×
28	from pants.engine.fs import (	×
29	CreateDigest,
30	Digest,
31	DigestSubset,
32	FileContent,
33	MergeDigests,
34	PathGlobs,
35	RemovePrefix,
36	)
37	from pants.engine.internals.graph import resolve_target	×
38	from pants.engine.internals.native_engine import Address	×
39	from pants.engine.intrinsics import create_digest, digest_to_snapshot, merge_digests, remove_prefix	×
40	from pants.engine.process import Process, fallible_to_exec_result_or_raise	×
41	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	×
42	from pants.engine.target import AllTargets, Target, Targets, WrappedTargetRequest	×
43	from pants.engine.unions import UnionMembership	×
44	from pants.util.docutil import doc_url	×
45	from pants.util.frozendict import FrozenDict	×
46	from pants.util.logging import LogLevel	×
47	from pants.util.osutil import is_macos_big_sur	×
48	from pants.util.resources import read_resource	×
49	from pants.util.strutil import softwrap	×
50
51	logger = logging.getLogger(__name__)	×
52
53
54	_scripts_package = "pants.backend.python.util_rules.scripts"	×
55
56
57	@dataclass(frozen=True)	×
58	class PEP660BuildResult:	×
59	output: Digest	×
60	# Relpaths in the output digest.
61	editable_wheel_path: str \| None	×
62
63
64	def dump_backend_wrapper_json(	×
65	dist_dir: str,
66	pth_file_path: str,
67	direct_url: str,
68	request: DistBuildRequest,
69	) -> bytes:
70	"""Build the settings json for our PEP 517 / PEP 660 wrapper script."""
71
72	def clean_config_settings(	×
73	cs: FrozenDict[str, tuple[str, ...]] \| None,
74	) -> dict[str, list[str]] \| None:
75	# setuptools.build_meta expects list values and chokes on tuples.
76	# We assume/hope that other backends accept lists as well.
77	return None if cs is None else {k: list(v) for k, v in cs.items()}	×
78
79	# tag the editable wheel as widely compatible
80	lang_tag, abi_tag, platform_tag = "py3", "none", "any"	×
81	if request.interpreter_constraints.includes_python2():	×
82	# Assume everything has py3 support. If not, we'll need a new includes_python3 method.
83	lang_tag = "py2.py3"	×
84
85	settings = {	×
86	"build_backend": request.build_system.build_backend,
87	"dist_dir": dist_dir,
88	"pth_file_path": pth_file_path,
89	"wheel_config_settings": clean_config_settings(request.wheel_config_settings),
90	"tags": "-".join([lang_tag, abi_tag, platform_tag]),
91	"direct_url": direct_url,
92	}
93	return json.dumps(settings).encode()	×
94
95
96	@rule	×
97	async def run_pep660_build(	×
98	request: DistBuildRequest, python_setup: PythonSetup, build_root: BuildRoot
99	) -> PEP660BuildResult:
100	"""Run our PEP 517 / PEP 660 wrapper script to generate an editable wheel.
101
102	The PEP 517 / PEP 660 wrapper script is responsible for building the editable wheel.
103	The backend wrapper script, along with the commands that install the editable wheel,
104	need to conform to the following specs so that Pants is a PEP 660 compliant frontend,
105	a PEP 660 compliant backend, and that it builds a compliant wheel and install.
106
107	NOTE: PEP 660 does not address the `.data` directory, so the wrapper ignores it.
108
109	Relevant Specs:
110	https://peps.python.org/pep-0517/
111	https://peps.python.org/pep-0660/
112	https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/
113	https://packaging.python.org/en/latest/specifications/recording-installed-packages/
114	https://packaging.python.org/en/latest/specifications/direct-url-data-structure/
115	https://packaging.python.org/en/latest/specifications/binary-distribution-format/
116	"""
117	dist_abs_path = (	×
118	build_root.path
119	if request.dist_source_root == "."
120	else str(build_root.pathlib_path / request.dist_source_root)
121	)
122	direct_url = "file://" + dist_abs_path.replace(os.path.sep, "/")	×
123
124	# Create the .pth files to add the relevant source root to sys.path.
125	# We cannot use the build backend to do this because we do not want to tell
126	# it where the workspace is and risk it adding anything there.
127	# NOTE: We use .pth files to support ICs less than python3.7.
128	# A future enhancement might be to provide more precise editable
129	# wheels based on https://pypi.org/project/editables/, but that only
130	# supports python3.7+ (what pip supports as of April 2023).
131	# Or maybe do something like setuptools strict editable wheel.
132	pth_file_contents = ""	×
133	for source_root in request.build_time_source_roots: # NB: the roots are sorted	×
134	# Can we use just the dist_abs_path instead of including all source roots?
135	abs_path = (	×
136	build_root.path if source_root == "." else str(build_root.pathlib_path / source_root)
137	)
138	pth_file_contents += f"{abs_path}\n"	×
139	pth_file_name = "__pants__.pth"	×
140	pth_file_path = os.path.join(request.working_directory, pth_file_name)	×
141
142	# This is the setuptools dist directory, not Pants's, so we hardcode to dist/.
143	dist_dir = "dist"	×
144	dist_output_dir = os.path.join(dist_dir, request.output_path)	×
145
146	backend_wrapper_json = "backend_wrapper.json"	×
147	backend_wrapper_json_path = os.path.join(request.working_directory, backend_wrapper_json)	×
148	backend_wrapper_name = "backend_wrapper.py"	×
149	backend_wrapper_path = os.path.join(request.working_directory, backend_wrapper_name)	×
150	backend_wrapper_content = read_resource(_scripts_package, "pep660_backend_wrapper.py")	×
151	assert backend_wrapper_content is not None	×
152
153	conf_digest, backend_wrapper_digest, build_backend_pex = await concurrently(	×
154	create_digest(
155	CreateDigest(
156	[
157	FileContent(pth_file_path, pth_file_contents.encode()),
158	FileContent(
159	backend_wrapper_json_path,
160	dump_backend_wrapper_json(
161	dist_output_dir, pth_file_name, direct_url, request
162	),
163	),
164	]
165	)
166	),
167	# The backend_wrapper has its own digest for cache reuse.
168	create_digest(CreateDigest([FileContent(backend_wrapper_path, backend_wrapper_content)])),
169	# Note that this pex has no entrypoint. We use it to run our wrapper, which
170	# in turn imports from and invokes the build backend.
171	create_venv_pex(
172	**implicitly(
173	PexRequest(
174	output_filename="build_backend.pex",
175	internal_only=True,
176	requirements=request.build_system.requires,
177	pex_path=request.extra_build_time_requirements,
178	interpreter_constraints=request.interpreter_constraints,
179	)
180	)
181	),
182	)
183
184	merged_digest = await merge_digests(	×
185	MergeDigests((request.input, conf_digest, backend_wrapper_digest))
186	)
187
188	extra_env = {	×
189	**(request.extra_build_time_env or {}),
190	"PEX_EXTRA_SYS_PATH": os.pathsep.join(request.build_time_source_roots),
191	}
192	if python_setup.macos_big_sur_compatibility and is_macos_big_sur():	×
193	extra_env["MACOSX_DEPLOYMENT_TARGET"] = "10.16"	×
194
195	result = await fallible_to_exec_result_or_raise(	×
196	**implicitly(
197	VenvPexProcess(
198	build_backend_pex,
199	argv=(backend_wrapper_name, backend_wrapper_json),
200	input_digest=merged_digest,
201	extra_env=extra_env,
202	working_directory=request.working_directory,
203	output_directories=(dist_dir,), # Relative to the working_directory.
204	description=(
205	f"Run {request.build_system.build_backend} to gather .dist-info for {request.target_address_spec}"
206	if request.target_address_spec
207	else f"Run {request.build_system.build_backend} to gather .dist-info"
208	),
209	level=LogLevel.DEBUG,
210	)
211	)
212	)
213	output_lines = result.stdout.decode().splitlines()	×
214	line_prefix = "editable_path: "	×
215	editable_path = ""	×
216	for line in output_lines:	×
217	if line.startswith(line_prefix):	×
218	editable_path = os.path.join(request.output_path, line[len(line_prefix) :].strip())	×
219	break	×
220
221	# Note that output_digest paths are relative to the working_directory.
222	output_digest = await remove_prefix(RemovePrefix(result.output_digest, dist_dir))	×
223	output_snapshot = await digest_to_snapshot(output_digest)	×
224	if editable_path not in output_snapshot.files:	×
225	raise BuildBackendError(	×
226	softwrap(
227	f"""
228	Failed to build PEP 660 editable wheel {editable_path}
229	(extracted dist-info from PEP 517 build backend
230	{request.build_system.build_backend}).
231	"""
232	)
233	)
234	return PEP660BuildResult(output_digest, editable_wheel_path=editable_path)	×
235
236
237	@dataclass(frozen=True)	×
238	class LocalDistPEP660Wheels:	×
239	"""Contains the PEP 660 "editable" wheels isolated from a single local Python distribution."""
240
241	pep660_wheel_paths: tuple[str, ...]	×
242	pep660_wheels_digest: Digest	×
243	provided_files: frozenset[str]	×
244
245
246	@rule	×
247	async def isolate_local_dist_pep660_wheels(	×
248	dist_target_address: Address,
249	bash: BashBinary,
250	unzip_binary: UnzipBinary,
251	python_setup: PythonSetup,
252	union_membership: UnionMembership,
253	) -> LocalDistPEP660Wheels:
254	dist_build_request = await create_dist_build_request(	×
255	dist_target_address=dist_target_address,
256	python_setup=python_setup,
257	union_membership=union_membership,
258	# editable wheel ignores build_wheel+build_sdist args
259	validate_wheel_sdist=False,
260	)
261	pep660_result = await run_pep660_build(dist_build_request, **implicitly())	×
262
263	# the output digest should only contain wheels, but filter to be safe.
264	wheels_snapshot = await digest_to_snapshot(	×
265	implicitly(DigestSubset(pep660_result.output, PathGlobs(["/*.whl"])))
266	)
267
268	wheels = tuple(sorted(wheels_snapshot.files))	×
269
270	if not wheels:	×
271	tgt = await resolve_target(	×
272	WrappedTargetRequest(dist_target_address, description_of_origin="<infallible>"),
273	**implicitly(),
274	)
275	logger.warning(	×
276	softwrap(
277	f"""
278	Encountered a dependency on the {tgt.target.alias} target at {dist_target_address},
279	but this target does not produce a Python wheel artifact. Therefore this target's
280	code will be used directly from sources, without a distribution being built,
281	and any native extensions in it will not be built.
282
283	See {doc_url("docs/python/overview/building-distributions")} for details on how to set up a
284	{tgt.target.alias} target to produce a wheel.
285	"""
286	)
287	)
288
289	wheels_listing_result = await fallible_to_exec_result_or_raise(	×
290	**implicitly(
291	Process(
292	argv=[
293	bash.path,
294	"-c",
295	f"""
296	set -ex
297	for f in {" ".join(shlex.quote(f) for f in wheels)}; do
298	{unzip_binary.path} -Z1 "$f"
299	done
300	""",
301	],
302	input_digest=wheels_snapshot.digest,
303	description=f"List contents of editable artifacts produced by {dist_target_address}",
304	)
305	)
306	)
307	provided_files = set(wheels_listing_result.stdout.decode().splitlines())	×
308
309	return LocalDistPEP660Wheels(wheels, wheels_snapshot.digest, frozenset(sorted(provided_files)))	×
310
311
312	@dataclass(frozen=True)	×
313	class AllPythonDistributionTargets:	×
314	targets: Targets	×
315
316
317	@rule(desc="Find all Python Distribution targets in project", level=LogLevel.DEBUG)	×
318	async def find_all_python_distributions(	×
319	all_targets: AllTargets,
320	) -> AllPythonDistributionTargets:
321	return AllPythonDistributionTargets(	×
322	# 'provides' is the field used in PythonDistributionFieldSet
323	Targets(tgt for tgt in all_targets if tgt.has_field(PythonProvidesField))
324	)
325
326
327	@dataclass(frozen=True)	×
328	class ResolveSortedPythonDistributionTargets:	×
329	targets: FrozenDict[str \| None, tuple[Target, ...]]	×
330
331
332	@rule(	×
333	desc="Associate resolves with all Python Distribution targets in project", level=LogLevel.DEBUG
334	)
335	async def sort_all_python_distributions_by_resolve(	×
336	all_dists: AllPythonDistributionTargets,
337	python_setup: PythonSetup,
338	) -> ResolveSortedPythonDistributionTargets:
339	dists = defaultdict(list)	×
340
341	if not python_setup.enable_resolves:	×
342	resolve = None	×
343	return ResolveSortedPythonDistributionTargets(	×
344	FrozenDict({resolve: tuple(all_dists.targets)})
345	)
346
347	dist_owned_deps = await concurrently(	×
348	get_owned_dependencies(DependencyOwner(ExportedTarget(tgt)), **implicitly())
349	for tgt in all_dists.targets
350	)
351
352	for dist, owned_deps in zip(all_dists.targets, dist_owned_deps):	×
353	resolve = None	×
354	# assumption: all owned deps are in the same resolve
355	for dep in owned_deps:	×
356	if dep.target.has_field(PythonResolveField):	×
357	resolve = dep.target[PythonResolveField].normalized_value(python_setup)	×
358	break	×
359	dists[resolve].append(dist)	×
360	return ResolveSortedPythonDistributionTargets(	×
361	FrozenDict({resolve: tuple(sorted(targets)) for resolve, targets in dists.items()})
362	)
363
364
365	@dataclass(frozen=True)	×
366	class EditableLocalDistsRequest:	×
367	"""Request to generate PEP660 wheels of local dists in the given resolve.
368
369	The editable wheel files must not be exported or made available to the end-user (according to
370	PEP 660). Instead, the PEP660 editable wheels serve as intermediate, internal-only,
371	representation of what should be installed in the exported virtualenv to create the editable
372	installs of local python_distributions.
373	"""
374
375	resolve: str \| None # None if resolves is not enabled	×
376
377
378	@dataclass(frozen=True)	×
379	class EditableLocalDists:	×
380	"""A Digest populated by editable (PEP660) wheels of local dists.
381
382	According to PEP660, these wheels should not be exported to users and must be discarded
383	after install. Anything that uses this should ensure that these wheels get installed and
384	then deleted.
385
386	Installing PEP660 wheels creates an "editable" install such that the sys.path gets
387	adjusted to include source directories from the build root (not from the sandbox).
388	This is decidedly not hermetic or portable and should only be used locally.
389
390	PEP660 wheels have .dist-info metadata and the .pth files (or similar) that adjust sys.path.
391	"""
392
393	optional_digest: Digest \| None	×
394
395
396	@rule(desc="Building editable local distributions (PEP 660)")	×
397	async def build_editable_local_dists(	×
398	request: EditableLocalDistsRequest,
399	all_dists: ResolveSortedPythonDistributionTargets,
400	python_setup: PythonSetup,
401	) -> EditableLocalDists:
402	resolve = request.resolve if python_setup.enable_resolves else None	×
403	resolve_dists = all_dists.targets.get(resolve, ())	×
404
405	if not resolve_dists:	×
406	return EditableLocalDists(None)	×
407
408	local_dists_wheels = await concurrently(	×
409	isolate_local_dist_pep660_wheels(target.address, **implicitly()) for target in resolve_dists
410	)
411
412	wheels: list[str] = []	×
413	wheels_digests = []	×
414	for local_dist_wheels in local_dists_wheels:	×
415	wheels.extend(local_dist_wheels.pep660_wheel_paths)	×
416	wheels_digests.append(local_dist_wheels.pep660_wheels_digest)	×
417
418	wheels_digest = await merge_digests(MergeDigests(wheels_digests))	×
419
420	return EditableLocalDists(wheels_digest)	×
421
422
423	def rules():	×
424	return (	×
425	*collect_rules(),
426	*dists_rules(),
427	*package_dists.rules(),
428	*system_binaries.rules(),
429	)

pantsbuild / pants / 18252174847

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous