18517631058

Committed 15 Oct 2025 04:18AM UTC coverage: 69.207% (-11.1%) from 80.267%

Build # 18517631058

Build Type

Pull #22745

github

Committed by

web-flow

Commit Message

Merge 642a76ca1 into 99919310e

Pull Request Pull Request #22745: [windows] Add windows support in the stdio crate.

Run Details

53815 of 77759 relevant lines covered (69.21%)

2.42 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

55.68

/src/python/pants/backend/python/util_rules/local_dists.py

# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import logging
import shlex
from collections.abc import Iterable
from dataclasses import dataclass

from pants.backend.python.subsystems.setuptools import PythonDistributionFieldSet
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.pex import Pex, PexRequest, create_pex
from pants.backend.python.util_rules.pex import rules as pex_rules
from pants.backend.python.util_rules.pex_requirements import PexRequirements
from pants.backend.python.util_rules.python_sources import PythonSourceFiles
from pants.build_graph.address import Address
from pants.core.goals.package import PackageFieldSet, build_package
from pants.core.util_rules import system_binaries
from pants.core.util_rules.source_files import SourceFiles
from pants.core.util_rules.system_binaries import BashBinary, UnzipBinary
from pants.engine.addresses import Addresses
from pants.engine.fs import Digest, DigestSubset, MergeDigests, PathGlobs
from pants.engine.internals.graph import resolve_target
from pants.engine.internals.graph import transitive_targets as transitive_targets_get
from pants.engine.intrinsics import digest_to_snapshot, merge_digests
from pants.engine.process import Process, fallible_to_exec_result_or_raise
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
from pants.engine.target import TransitiveTargetsRequest, WrappedTargetRequest
from pants.util.dirutil import fast_relpath_optional
from pants.util.docutil import doc_url
from pants.util.strutil import softwrap

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class LocalDistWheels:
    """Contains the wheels isolated from a single local Python distribution."""

    wheel_paths: tuple[str, ...]
    wheels_digest: Digest
    provided_files: frozenset[str]


@rule
async def isolate_local_dist_wheels(
    dist_field_set: PythonDistributionFieldSet,
    bash: BashBinary,
    unzip_binary: UnzipBinary,
) -> LocalDistWheels:
    dist = await build_package(**implicitly({dist_field_set: PackageFieldSet}))
    wheels_snapshot = await digest_to_snapshot(
        **implicitly(DigestSubset(dist.digest, PathGlobs(["**/*.whl"])))
    )

    # A given local dist might build a wheel and an sdist (and maybe other artifacts -
    # we don't know what setup command was run...)
    # As long as there is a wheel, we can ignore the other artifacts.
    artifacts = {(a.relpath or "") for a in dist.artifacts}
    wheels = [wheel for wheel in wheels_snapshot.files if wheel in artifacts]

    if not wheels:
        tgt = await resolve_target(
            WrappedTargetRequest(dist_field_set.address, description_of_origin="<infallible>"),
            **implicitly(),
        )
        logger.warning(
            softwrap(
                f"""
                Encountered a dependency on the {tgt.target.alias} target at {dist_field_set.address},
                but this target does not produce a Python wheel artifact. Therefore this target's
                code will be used directly from sources, without a distribution being built,
                and any native extensions in it will not be built.

                See {doc_url("docs/python/overview/building-distributions")} for details on how to set up a
                {tgt.target.alias} target to produce a wheel.
                """
            )
        )

    wheels_listing_result = await fallible_to_exec_result_or_raise(
        **implicitly(
            Process(
                argv=[
                    bash.path,
                    "-c",
                    f"""
                set -ex
                for f in {" ".join(shlex.quote(f) for f in wheels)}; do
                  {unzip_binary.path} -Z1 "$f"
                done
                """,
                ],
                input_digest=wheels_snapshot.digest,
                description=f"List contents of artifacts produced by {dist_field_set.address}",
            )
        )
    )
    provided_files = set(wheels_listing_result.stdout.decode().splitlines())

    return LocalDistWheels(
        tuple(sorted(wheels)), wheels_snapshot.digest, frozenset(sorted(provided_files))
    )


@dataclass(frozen=True)
class LocalDistsPexRequest:
    """Request to build the local dists from the dependency closure of a set of addresses."""

    addresses: Addresses
    interpreter_constraints: InterpreterConstraints
    # The result will return these with the sources provided by the dists subtracted out.
    # This will help the caller prevent sources from appearing twice on sys.path.
    sources: PythonSourceFiles

    def __init__(
        self,
        addresses: Iterable[Address],
        *,
        interpreter_constraints: InterpreterConstraints,
        sources: PythonSourceFiles = PythonSourceFiles.empty(),
    ) -> None:
        object.__setattr__(self, "addresses", Addresses(addresses))
        object.__setattr__(self, "interpreter_constraints", interpreter_constraints)
        object.__setattr__(self, "sources", sources)


@dataclass(frozen=True)
class LocalDistsPex:
    """A PEX file containing locally-built dists.

    Can be consumed from another PEX, e.g., by adding to PEX_PATH.

    The PEX will only contain locally built dists and not their dependencies. For Pants generated
    `setup.py` / `pyproject.toml`, the dependencies will be included in the standard resolve process
    that the locally-built dists PEX is adjoined to via PEX_PATH. For hand-made `setup.py` /
    `pyproject.toml` with 3rdparty dependencies not hand-mirrored into BUILD file dependencies, this
    will lead to issues. See https://github.com/pantsbuild/pants/issues/13587#issuecomment-974863636
    for one way to fix this corner which is intentionally punted on for now.

    Lists the files provided by the dists on sys.path, so they can be subtracted from
    sources digests, to prevent the same file ending up on sys.path twice.
    """

    pex: Pex
    # The sources from the request, but with any files provided by the local dists subtracted out.
    remaining_sources: PythonSourceFiles


@rule(desc="Building local distributions")
async def build_local_dists(
    request: LocalDistsPexRequest,
) -> LocalDistsPex:
    transitive_targets = await transitive_targets_get(
        TransitiveTargetsRequest(request.addresses), **implicitly()
    )
    applicable_targets = [
        tgt for tgt in transitive_targets.closure if PythonDistributionFieldSet.is_applicable(tgt)
    ]

    local_dists_wheels = await concurrently(
        isolate_local_dist_wheels(PythonDistributionFieldSet.create(target), **implicitly())
        for target in applicable_targets
    )

    # The primary use-case of the "local dists" feature is to support consuming native extensions
    # as wheels without having to publish them first.
    # It doesn't seem very useful to consume locally-built sdists, and it makes it hard to
    # reason about possible sys.path collisions between the in-repo sources and whatever the
    # sdist will place on the sys.path when it's installed.
    # So for now we simply ignore sdists, with a warning if necessary.
    provided_files: set[str] = set()
    wheels: list[str] = []
    wheels_digests = []
    for local_dist_wheels in local_dists_wheels:
        wheels.extend(local_dist_wheels.wheel_paths)
        wheels_digests.append(local_dist_wheels.wheels_digest)
        provided_files.update(local_dist_wheels.provided_files)

    wheels_digest = await merge_digests(MergeDigests(wheels_digests))

    dists_pex = await create_pex(
        PexRequest(
            output_filename="local_dists.pex",
            requirements=PexRequirements(wheels),
            interpreter_constraints=request.interpreter_constraints,
            additional_inputs=wheels_digest,
            # a "local dists" PEX is always just for consumption by some downstream Pants process,
            # i.e. internal
            internal_only=True,
            additional_args=["--intransitive"],
        )
    )

    if not wheels:
        # The source calculations below are not (always) cheap, so we skip them if no wheels were
        # produced. See https://github.com/pantsbuild/pants/issues/14561 for one possible approach
        # to sharing the cost of these calculations.
        return LocalDistsPex(dists_pex, request.sources)

    # We check source roots in reverse lexicographic order,
    # so we'll find the innermost root that matches.
    source_roots = sorted(request.sources.source_roots, reverse=True)
    remaining_sources = set(request.sources.source_files.files)
    unrooted_files_set = set(request.sources.source_files.unrooted_files)
    for source in request.sources.source_files.files:
        if source not in unrooted_files_set:
            for source_root in source_roots:
                source_relpath = fast_relpath_optional(source, source_root)
                if source_relpath is not None and source_relpath in provided_files:
                    remaining_sources.remove(source)
    remaining_sources_snapshot = await digest_to_snapshot(
        **implicitly(
            DigestSubset(
                request.sources.source_files.snapshot.digest, PathGlobs(sorted(remaining_sources))
            )
        )
    )
    subtracted_sources = PythonSourceFiles(
        SourceFiles(remaining_sources_snapshot, request.sources.source_files.unrooted_files),
        request.sources.source_roots,
    )

    return LocalDistsPex(dists_pex, subtracted_sources)


def rules():
    return (
        *collect_rules(),
        *pex_rules(),
        *system_binaries.rules(),
    )

1	# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	7✔
5
6	import logging	7✔
7	import shlex	7✔
8	from collections.abc import Iterable	7✔
9	from dataclasses import dataclass	7✔
10
11	from pants.backend.python.subsystems.setuptools import PythonDistributionFieldSet	7✔
12	from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints	7✔
13	from pants.backend.python.util_rules.pex import Pex, PexRequest, create_pex	7✔
14	from pants.backend.python.util_rules.pex import rules as pex_rules	7✔
15	from pants.backend.python.util_rules.pex_requirements import PexRequirements	7✔
16	from pants.backend.python.util_rules.python_sources import PythonSourceFiles	7✔
17	from pants.build_graph.address import Address	7✔
18	from pants.core.goals.package import PackageFieldSet, build_package	7✔
19	from pants.core.util_rules import system_binaries	7✔
20	from pants.core.util_rules.source_files import SourceFiles	7✔
21	from pants.core.util_rules.system_binaries import BashBinary, UnzipBinary	7✔
22	from pants.engine.addresses import Addresses	7✔
23	from pants.engine.fs import Digest, DigestSubset, MergeDigests, PathGlobs	7✔
24	from pants.engine.internals.graph import resolve_target	7✔
25	from pants.engine.internals.graph import transitive_targets as transitive_targets_get	7✔
26	from pants.engine.intrinsics import digest_to_snapshot, merge_digests	7✔
27	from pants.engine.process import Process, fallible_to_exec_result_or_raise	7✔
28	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	7✔
29	from pants.engine.target import TransitiveTargetsRequest, WrappedTargetRequest	7✔
30	from pants.util.dirutil import fast_relpath_optional	7✔
31	from pants.util.docutil import doc_url	7✔
32	from pants.util.strutil import softwrap	7✔
33
34	logger = logging.getLogger(__name__)	7✔
35
36
37	@dataclass(frozen=True)	7✔
38	class LocalDistWheels:	7✔
39	"""Contains the wheels isolated from a single local Python distribution."""
40
41	wheel_paths: tuple[str, ...]	7✔
42	wheels_digest: Digest	7✔
43	provided_files: frozenset[str]	7✔
44
45
46	@rule	7✔
47	async def isolate_local_dist_wheels(	7✔
48	dist_field_set: PythonDistributionFieldSet,
49	bash: BashBinary,
50	unzip_binary: UnzipBinary,
51	) -> LocalDistWheels:
52	dist = await build_package(**implicitly({dist_field_set: PackageFieldSet}))	×
53	wheels_snapshot = await digest_to_snapshot(	×
54	implicitly(DigestSubset(dist.digest, PathGlobs(["/*.whl"])))
55	)
56
57	# A given local dist might build a wheel and an sdist (and maybe other artifacts -
58	# we don't know what setup command was run...)
59	# As long as there is a wheel, we can ignore the other artifacts.
60	artifacts = {(a.relpath or "") for a in dist.artifacts}	×
61	wheels = [wheel for wheel in wheels_snapshot.files if wheel in artifacts]	×
62
63	if not wheels:	×
64	tgt = await resolve_target(	×
65	WrappedTargetRequest(dist_field_set.address, description_of_origin="<infallible>"),
66	**implicitly(),
67	)
68	logger.warning(	×
69	softwrap(
70	f"""
71	Encountered a dependency on the {tgt.target.alias} target at {dist_field_set.address},
72	but this target does not produce a Python wheel artifact. Therefore this target's
73	code will be used directly from sources, without a distribution being built,
74	and any native extensions in it will not be built.
75
76	See {doc_url("docs/python/overview/building-distributions")} for details on how to set up a
77	{tgt.target.alias} target to produce a wheel.
78	"""
79	)
80	)
81
82	wheels_listing_result = await fallible_to_exec_result_or_raise(	×
83	**implicitly(
84	Process(
85	argv=[
86	bash.path,
87	"-c",
88	f"""
89	set -ex
90	for f in {" ".join(shlex.quote(f) for f in wheels)}; do
91	{unzip_binary.path} -Z1 "$f"
92	done
93	""",
94	],
95	input_digest=wheels_snapshot.digest,
96	description=f"List contents of artifacts produced by {dist_field_set.address}",
97	)
98	)
99	)
100	provided_files = set(wheels_listing_result.stdout.decode().splitlines())	×
101
102	return LocalDistWheels(	×
103	tuple(sorted(wheels)), wheels_snapshot.digest, frozenset(sorted(provided_files))
104	)
105
106
107	@dataclass(frozen=True)	7✔
108	class LocalDistsPexRequest:	7✔
109	"""Request to build the local dists from the dependency closure of a set of addresses."""
110
111	addresses: Addresses	7✔
112	interpreter_constraints: InterpreterConstraints	7✔
113	# The result will return these with the sources provided by the dists subtracted out.
114	# This will help the caller prevent sources from appearing twice on sys.path.
115	sources: PythonSourceFiles	7✔
116
117	def __init__(	7✔
118	self,
119	addresses: Iterable[Address],
120	*,
121	interpreter_constraints: InterpreterConstraints,
122	sources: PythonSourceFiles = PythonSourceFiles.empty(),
123	) -> None:
124	object.__setattr__(self, "addresses", Addresses(addresses))	×
125	object.__setattr__(self, "interpreter_constraints", interpreter_constraints)	×
126	object.__setattr__(self, "sources", sources)	×
127
128
129	@dataclass(frozen=True)	7✔
130	class LocalDistsPex:	7✔
131	"""A PEX file containing locally-built dists.
132
133	Can be consumed from another PEX, e.g., by adding to PEX_PATH.
134
135	The PEX will only contain locally built dists and not their dependencies. For Pants generated
136	`setup.py` / `pyproject.toml`, the dependencies will be included in the standard resolve process
137	that the locally-built dists PEX is adjoined to via PEX_PATH. For hand-made `setup.py` /
138	`pyproject.toml` with 3rdparty dependencies not hand-mirrored into BUILD file dependencies, this
139	will lead to issues. See https://github.com/pantsbuild/pants/issues/13587#issuecomment-974863636
140	for one way to fix this corner which is intentionally punted on for now.
141
142	Lists the files provided by the dists on sys.path, so they can be subtracted from
143	sources digests, to prevent the same file ending up on sys.path twice.
144	"""
145
146	pex: Pex	7✔
147	# The sources from the request, but with any files provided by the local dists subtracted out.
148	remaining_sources: PythonSourceFiles	7✔
149
150
151	@rule(desc="Building local distributions")	7✔
152	async def build_local_dists(	7✔
153	request: LocalDistsPexRequest,
154	) -> LocalDistsPex:
155	transitive_targets = await transitive_targets_get(	×
156	TransitiveTargetsRequest(request.addresses), **implicitly()
157	)
158	applicable_targets = [	×
159	tgt for tgt in transitive_targets.closure if PythonDistributionFieldSet.is_applicable(tgt)
160	]
161
162	local_dists_wheels = await concurrently(	×
163	isolate_local_dist_wheels(PythonDistributionFieldSet.create(target), **implicitly())
164	for target in applicable_targets
165	)
166
167	# The primary use-case of the "local dists" feature is to support consuming native extensions
168	# as wheels without having to publish them first.
169	# It doesn't seem very useful to consume locally-built sdists, and it makes it hard to
170	# reason about possible sys.path collisions between the in-repo sources and whatever the
171	# sdist will place on the sys.path when it's installed.
172	# So for now we simply ignore sdists, with a warning if necessary.
173	provided_files: set[str] = set()	×
174	wheels: list[str] = []	×
175	wheels_digests = []	×
176	for local_dist_wheels in local_dists_wheels:	×
177	wheels.extend(local_dist_wheels.wheel_paths)	×
178	wheels_digests.append(local_dist_wheels.wheels_digest)	×
179	provided_files.update(local_dist_wheels.provided_files)	×
180
181	wheels_digest = await merge_digests(MergeDigests(wheels_digests))	×
182
183	dists_pex = await create_pex(	×
184	PexRequest(
185	output_filename="local_dists.pex",
186	requirements=PexRequirements(wheels),
187	interpreter_constraints=request.interpreter_constraints,
188	additional_inputs=wheels_digest,
189	# a "local dists" PEX is always just for consumption by some downstream Pants process,
190	# i.e. internal
191	internal_only=True,
192	additional_args=["--intransitive"],
193	)
194	)
195
196	if not wheels:	×
197	# The source calculations below are not (always) cheap, so we skip them if no wheels were
198	# produced. See https://github.com/pantsbuild/pants/issues/14561 for one possible approach
199	# to sharing the cost of these calculations.
200	return LocalDistsPex(dists_pex, request.sources)	×
201
202	# We check source roots in reverse lexicographic order,
203	# so we'll find the innermost root that matches.
204	source_roots = sorted(request.sources.source_roots, reverse=True)	×
205	remaining_sources = set(request.sources.source_files.files)	×
206	unrooted_files_set = set(request.sources.source_files.unrooted_files)	×
207	for source in request.sources.source_files.files:	×
208	if source not in unrooted_files_set:	×
209	for source_root in source_roots:	×
210	source_relpath = fast_relpath_optional(source, source_root)	×
211	if source_relpath is not None and source_relpath in provided_files:	×
212	remaining_sources.remove(source)	×
213	remaining_sources_snapshot = await digest_to_snapshot(	×
214	**implicitly(
215	DigestSubset(
216	request.sources.source_files.snapshot.digest, PathGlobs(sorted(remaining_sources))
217	)
218	)
219	)
220	subtracted_sources = PythonSourceFiles(	×
221	SourceFiles(remaining_sources_snapshot, request.sources.source_files.unrooted_files),
222	request.sources.source_roots,
223	)
224
225	return LocalDistsPex(dists_pex, subtracted_sources)	×
226
227
228	def rules():	7✔
229	return (	6✔
230	*collect_rules(),
231	*pex_rules(),
232	*system_binaries.rules(),
233	)

pantsbuild / pants / 18517631058

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous