18791134616

Committed 24 Oct 2025 08:18PM UTC coverage: 75.519% (-4.8%) from 80.282%

Build # 18791134616

Build Type

Pull #22794

github

Committed by

web-flow

Commit Message

Merge 098c595a0 into 7971a20bf

Pull Request Pull Request #22794: Use self-hosted MacOS Intel runner

Run Details

65803 of 87134 relevant lines covered (75.52%)

3.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

53.85

/src/python/pants/core/goals/export.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import itertools
import os
from collections import defaultdict
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import cast

from pants.base.build_root import BuildRoot
from pants.core.goals.generate_lockfiles import (
    KnownUserResolveNamesRequest,
    UnrecognizedResolveNamesError,
    get_known_user_resolve_names,
)
from pants.core.goals.resolves import ExportableTool, ExportMode
from pants.core.util_rules.distdir import DistDir
from pants.core.util_rules.env_vars import environment_vars_subset
from pants.engine.collection import Collection
from pants.engine.console import Console
from pants.engine.env_vars import EnvironmentVarsRequest
from pants.engine.environment import EnvironmentName
from pants.engine.fs import (
    EMPTY_DIGEST,
    AddPrefix,
    CreateDigest,
    Digest,
    MergeDigests,
    SymlinkEntry,
    Workspace,
)
from pants.engine.goal import Goal, GoalSubsystem
from pants.engine.internals.selectors import concurrently
from pants.engine.intrinsics import (
    add_prefix,
    create_digest,
    merge_digests,
    run_interactive_process,
)
from pants.engine.process import InteractiveProcess
from pants.engine.rules import collect_rules, goal_rule, implicitly, rule
from pants.engine.target import FilteredTargets, Target
from pants.engine.unions import UnionMembership, union
from pants.option.option_types import StrListOption
from pants.util.dirutil import safe_mkdir, safe_rmtree
from pants.util.frozendict import FrozenDict
from pants.util.strutil import softwrap


class ExportError(Exception):
    pass


@union(in_scope_types=[EnvironmentName])
@dataclass(frozen=True)
class ExportRequest:
    """A union for exportable data provided by a backend.

    Subclass and install a member of this type to export data.
    """

    targets: Sequence[Target]


@dataclass(frozen=True)
class PostProcessingCommand:
    """A command to run as a local process after an exported digest is materialized."""

    # Values in the argv tuple can contain the format specifier "{digest_root}", which will be
    # substituted with the (absolute) path to the location under distdir in which the
    # digest is materialized.
    argv: tuple[str, ...]
    # The command will be run with an environment consisting of just PATH, set to the Pants
    # process's own PATH env var, plus these extra env vars.
    extra_env: FrozenDict[str, str]

    def __init__(
        self,
        argv: Iterable[str],
        extra_env: Mapping[str, str] = FrozenDict(),
    ):
        object.__setattr__(self, "argv", tuple(argv))
        object.__setattr__(self, "extra_env", FrozenDict(extra_env))


@dataclass(frozen=True)
class ExportedBinary:
    """Binaries exposed by an export.

    These will be added under the "bin" folder. The `name` is the name that will be linked as in the
    `bin` folder. The `path_in_export` is the path within the exported digest to link to. These can
    be used to abstract details from the name of the tool and avoid the other files in the tool's
    digest.

    For example, "my-tool" might have a downloaded file of
    "my_tool/my_tool_linux_x86-64.bin" and a readme. We would use `ExportedBinary(name="my-tool",
    path_in_export=my_tool/my_tool_linux_x86-64.bin"`
    """

    name: str
    path_in_export: str


@dataclass(frozen=True)
class ExportResult:
    description: str
    # Materialize digests under this reldir.
    reldir: str
    # Materialize this digest.
    digest: Digest
    # Run these commands as local processes after the digest is materialized.
    post_processing_cmds: tuple[PostProcessingCommand, ...]
    # Set for the common special case of exporting a resolve, and names that resolve.
    # Set to None for other export results.
    resolve: str | None
    exported_binaries: tuple[ExportedBinary, ...]

    def __init__(
        self,
        description: str,
        reldir: str,
        *,
        digest: Digest = EMPTY_DIGEST,
        post_processing_cmds: Iterable[PostProcessingCommand] = tuple(),
        resolve: str | None = None,
        exported_binaries: Iterable[ExportedBinary] = tuple(),
    ):
        object.__setattr__(self, "description", description)
        object.__setattr__(self, "reldir", reldir)
        object.__setattr__(self, "digest", digest)
        object.__setattr__(self, "post_processing_cmds", tuple(post_processing_cmds))
        object.__setattr__(self, "resolve", resolve)
        object.__setattr__(self, "exported_binaries", tuple(exported_binaries))


class ExportResults(Collection[ExportResult]):
    pass


@rule(polymorphic=True)
async def export(
    req: ExportRequest,
    environment_name: EnvironmentName,
) -> ExportResults:
    raise NotImplementedError()


class ExportSubsystem(GoalSubsystem):
    name = "export"
    help = softwrap(
        """
        Export Pants data for use in other tools, such as IDEs.

        :::caution Exporting tools requires creating a custom lockfile for them

        Follow [the instructions for creating tool lockfiles](../../docs/python/overview/lockfiles#lockfiles-for-tools)

        :::
        """
    )

    # NB: Only options that are relevant across many/most backends and languages
    #  should be defined here.  Backend-specific options should be defined in that backend
    #  as plugin options on this subsystem.

    # Exporting resolves is a common use-case for `export`, often the primary one, so we
    # add affordances for it at the core goal level.
    resolve = StrListOption(
        default=[],
        help="Export the specified resolve(s). The export format is backend-specific, "
        "e.g., Python resolves are exported as virtualenvs.",
    )

    binaries = StrListOption(
        flag_name="--bin",  # `bin` is a python builtin
        default=[],
        help="Export the specified binaries. To select a binary, provide its subsystem scope name, as used for setting its options.",
    )


class Export(Goal):
    subsystem_cls = ExportSubsystem
    environment_behavior = Goal.EnvironmentBehavior.LOCAL_ONLY


@goal_rule
async def export_goal(
    console: Console,
    targets: FilteredTargets,
    workspace: Workspace,
    union_membership: UnionMembership,
    build_root: BuildRoot,
    dist_dir: DistDir,
    export_subsys: ExportSubsystem,
) -> Export:
    request_types = cast("Iterable[type[ExportRequest]]", union_membership.get(ExportRequest))

    if not (export_subsys.resolve or export_subsys.options.bin):
        raise ExportError("Must specify at least one `--resolve` or `--bin` to export")
    if targets:
        raise ExportError("The `export` goal does not take target specs.")

    requests = tuple(request_type(targets) for request_type in request_types)
    all_results = await concurrently(
        export(**implicitly({request: ExportRequest})) for request in requests
    )
    flattened_results = sorted(
        (res for results in all_results for res in results), key=lambda res: res.resolve or ""
    )  # sorting provides predictable resolution in conflicts

    prefixed_digests = await concurrently(
        add_prefix(AddPrefix(result.digest, result.reldir)) for result in flattened_results
    )
    output_dir = os.path.join(str(dist_dir.relpath), "export")
    for result in flattened_results:
        digest_root = os.path.join(build_root.path, output_dir, result.reldir)
        safe_rmtree(digest_root)
    merged_digest = await merge_digests(MergeDigests(prefixed_digests))
    dist_digest = await add_prefix(AddPrefix(merged_digest, output_dir))
    workspace.write_digest(dist_digest)
    environment = await environment_vars_subset(EnvironmentVarsRequest(["PATH"]), **implicitly())
    resolves_exported = set()
    for result in flattened_results:
        result_dir = os.path.join(output_dir, result.reldir)
        digest_root = os.path.join(build_root.path, result_dir)
        for cmd in result.post_processing_cmds:
            argv = tuple(arg.format(digest_root=digest_root) for arg in cmd.argv)
            ip = InteractiveProcess(
                argv=argv,
                env={"PATH": environment.get("PATH", ""), **cmd.extra_env},
                run_in_workspace=True,
            )
            ipr = await run_interactive_process(ip)
            if ipr.exit_code:
                raise ExportError(f"Failed to write {result.description} to {result_dir}")
        if result.resolve:
            resolves_exported.add(result.resolve)
        console.print_stdout(f"Wrote {result.description} to {result_dir}")

    exported_bins_by_exporting_resolve, link_requests = await link_exported_executables(
        build_root, output_dir, flattened_results
    )
    link_digest = await create_digest(link_requests)
    workspace.write_digest(link_digest)

    exported_bin_warnings = warn_exported_bin_conflicts(exported_bins_by_exporting_resolve)
    for warning in exported_bin_warnings:
        console.print_stderr(warning)

    unexported_resolves = sorted(
        (set(export_subsys.resolve) | set(export_subsys.binaries)) - resolves_exported
    )
    if unexported_resolves:
        all_known_user_resolve_names = await concurrently(
            get_known_user_resolve_names(**implicitly({request(): KnownUserResolveNamesRequest}))
            for request in union_membership.get(KnownUserResolveNamesRequest)
        )
        all_known_bin_names = [
            e.options_scope
            for e in union_membership.get(ExportableTool)
            if e.export_mode == ExportMode.binary
        ]
        all_valid_resolve_names = sorted(
            {
                *itertools.chain.from_iterable(kurn.names for kurn in all_known_user_resolve_names),
            }
        )
        raise UnrecognizedResolveNamesError(
            unexported_resolves,
            all_valid_resolve_names,
            all_known_bin_names,
            description_of_origin="the options --export-resolve and/or --export-bin",
        )

    return Export(exit_code=0)


async def link_exported_executables(
    build_root: BuildRoot,
    output_dir: str,
    export_results: list[ExportResult],
) -> tuple[dict[str, list[str]], CreateDigest]:
    """Link the exported executables to the `bin` dir.

    Multiple resolves might export the same executable. This will export the first one only but
    track the collision.
    """
    safe_mkdir(Path(output_dir, "bin"))

    exported_bins_by_exporting_resolve: dict[str, list[str]] = defaultdict(list)
    link_requests = []
    for result in export_results:
        for exported_bin in result.exported_binaries:
            exported_bins_by_exporting_resolve[exported_bin.name].append(
                result.resolve or result.description
            )
            if len(exported_bins_by_exporting_resolve[exported_bin.name]) > 1:
                continue

            path = Path(output_dir, "bin", exported_bin.name)
            target = Path(build_root.path, output_dir, result.reldir, exported_bin.path_in_export)
            link_requests.append(SymlinkEntry(path.as_posix(), target.as_posix()))

    return exported_bins_by_exporting_resolve, CreateDigest(link_requests)


def warn_exported_bin_conflicts(exported_bins: dict[str, list[str]]) -> list[str]:
    """Check that no bin was exported from multiple resolves."""
    messages = []

    for exported_bin_name, resolves in exported_bins.items():
        if len(resolves) > 1:
            msg = f"Exporting binary `{exported_bin_name}` had conflicts. "
            succeeded_resolve, other_resolves = resolves[0], resolves[1:]
            msg += (
                f"The resolve {succeeded_resolve} was exported, but it conflicted with "
                + ", ".join(other_resolves)
            )
            messages.append(msg)

    return messages


def rules():
    return collect_rules()

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	10✔
5
6	import itertools	10✔
7	import os	10✔
8	from collections import defaultdict	10✔
9	from collections.abc import Iterable, Mapping, Sequence	10✔
10	from dataclasses import dataclass	10✔
11	from pathlib import Path	10✔
12	from typing import cast	10✔
13
14	from pants.base.build_root import BuildRoot	10✔
15	from pants.core.goals.generate_lockfiles import (	10✔
16	KnownUserResolveNamesRequest,
17	UnrecognizedResolveNamesError,
18	get_known_user_resolve_names,
19	)
20	from pants.core.goals.resolves import ExportableTool, ExportMode	10✔
21	from pants.core.util_rules.distdir import DistDir	10✔
22	from pants.core.util_rules.env_vars import environment_vars_subset	10✔
23	from pants.engine.collection import Collection	10✔
24	from pants.engine.console import Console	10✔
25	from pants.engine.env_vars import EnvironmentVarsRequest	10✔
26	from pants.engine.environment import EnvironmentName	10✔
27	from pants.engine.fs import (	10✔
28	EMPTY_DIGEST,
29	AddPrefix,
30	CreateDigest,
31	Digest,
32	MergeDigests,
33	SymlinkEntry,
34	Workspace,
35	)
36	from pants.engine.goal import Goal, GoalSubsystem	10✔
37	from pants.engine.internals.selectors import concurrently	10✔
38	from pants.engine.intrinsics import (	10✔
39	add_prefix,
40	create_digest,
41	merge_digests,
42	run_interactive_process,
43	)
44	from pants.engine.process import InteractiveProcess	10✔
45	from pants.engine.rules import collect_rules, goal_rule, implicitly, rule	10✔
46	from pants.engine.target import FilteredTargets, Target	10✔
47	from pants.engine.unions import UnionMembership, union	10✔
48	from pants.option.option_types import StrListOption	10✔
49	from pants.util.dirutil import safe_mkdir, safe_rmtree	10✔
50	from pants.util.frozendict import FrozenDict	10✔
51	from pants.util.strutil import softwrap	10✔
52
53
54	class ExportError(Exception):	10✔
55	pass	10✔
56
57
58	@union(in_scope_types=[EnvironmentName])	10✔
59	@dataclass(frozen=True)	10✔
60	class ExportRequest:	10✔
61	"""A union for exportable data provided by a backend.
62
63	Subclass and install a member of this type to export data.
64	"""
65
66	targets: Sequence[Target]	10✔
67
68
69	@dataclass(frozen=True)	10✔
70	class PostProcessingCommand:	10✔
71	"""A command to run as a local process after an exported digest is materialized."""
72
73	# Values in the argv tuple can contain the format specifier "{digest_root}", which will be
74	# substituted with the (absolute) path to the location under distdir in which the
75	# digest is materialized.
76	argv: tuple[str, ...]	10✔
77	# The command will be run with an environment consisting of just PATH, set to the Pants
78	# process's own PATH env var, plus these extra env vars.
79	extra_env: FrozenDict[str, str]	10✔
80
81	def __init__(	10✔
82	self,
83	argv: Iterable[str],
84	extra_env: Mapping[str, str] = FrozenDict(),
85	):
86	object.__setattr__(self, "argv", tuple(argv))	×
87	object.__setattr__(self, "extra_env", FrozenDict(extra_env))	×
88
89
90	@dataclass(frozen=True)	10✔
91	class ExportedBinary:	10✔
92	"""Binaries exposed by an export.
93
94	These will be added under the "bin" folder. The `name` is the name that will be linked as in the
95	`bin` folder. The `path_in_export` is the path within the exported digest to link to. These can
96	be used to abstract details from the name of the tool and avoid the other files in the tool's
97	digest.
98
99	For example, "my-tool" might have a downloaded file of
100	"my_tool/my_tool_linux_x86-64.bin" and a readme. We would use `ExportedBinary(name="my-tool",
101	path_in_export=my_tool/my_tool_linux_x86-64.bin"`
102	"""
103
104	name: str	10✔
105	path_in_export: str	10✔
106
107
108	@dataclass(frozen=True)	10✔
109	class ExportResult:	10✔
110	description: str	10✔
111	# Materialize digests under this reldir.
112	reldir: str	10✔
113	# Materialize this digest.
114	digest: Digest	10✔
115	# Run these commands as local processes after the digest is materialized.
116	post_processing_cmds: tuple[PostProcessingCommand, ...]	10✔
117	# Set for the common special case of exporting a resolve, and names that resolve.
118	# Set to None for other export results.
119	resolve: str \| None	10✔
120	exported_binaries: tuple[ExportedBinary, ...]	10✔
121
122	def __init__(	10✔
123	self,
124	description: str,
125	reldir: str,
126	*,
127	digest: Digest = EMPTY_DIGEST,
128	post_processing_cmds: Iterable[PostProcessingCommand] = tuple(),
129	resolve: str \| None = None,
130	exported_binaries: Iterable[ExportedBinary] = tuple(),
131	):
132	object.__setattr__(self, "description", description)	1✔
133	object.__setattr__(self, "reldir", reldir)	1✔
134	object.__setattr__(self, "digest", digest)	1✔
135	object.__setattr__(self, "post_processing_cmds", tuple(post_processing_cmds))	1✔
136	object.__setattr__(self, "resolve", resolve)	1✔
137	object.__setattr__(self, "exported_binaries", tuple(exported_binaries))	1✔
138
139
140	class ExportResults(Collection[ExportResult]):	10✔
141	pass	10✔
142
143
144	@rule(polymorphic=True)	10✔
145	async def export(	10✔
146	req: ExportRequest,
147	environment_name: EnvironmentName,
148	) -> ExportResults:
149	raise NotImplementedError()	×
150
151
152	class ExportSubsystem(GoalSubsystem):	10✔
153	name = "export"	10✔
154	help = softwrap(	10✔
155	"""
156	Export Pants data for use in other tools, such as IDEs.
157
158	:::caution Exporting tools requires creating a custom lockfile for them
159
160	Follow [the instructions for creating tool lockfiles](../../docs/python/overview/lockfiles#lockfiles-for-tools)
161
162	:::
163	"""
164	)
165
166	# NB: Only options that are relevant across many/most backends and languages
167	# should be defined here. Backend-specific options should be defined in that backend
168	# as plugin options on this subsystem.
169
170	# Exporting resolves is a common use-case for `export`, often the primary one, so we
171	# add affordances for it at the core goal level.
172	resolve = StrListOption(	10✔
173	default=[],
174	help="Export the specified resolve(s). The export format is backend-specific, "
175	"e.g., Python resolves are exported as virtualenvs.",
176	)
177
178	binaries = StrListOption(	10✔
179	flag_name="--bin", # `bin` is a python builtin
180	default=[],
181	help="Export the specified binaries. To select a binary, provide its subsystem scope name, as used for setting its options.",
182	)
183
184
185	class Export(Goal):	10✔
186	subsystem_cls = ExportSubsystem	10✔
187	environment_behavior = Goal.EnvironmentBehavior.LOCAL_ONLY	10✔
188
189
190	@goal_rule	10✔
191	async def export_goal(	10✔
192	console: Console,
193	targets: FilteredTargets,
194	workspace: Workspace,
195	union_membership: UnionMembership,
196	build_root: BuildRoot,
197	dist_dir: DistDir,
198	export_subsys: ExportSubsystem,
199	) -> Export:
200	request_types = cast("Iterable[type[ExportRequest]]", union_membership.get(ExportRequest))	×
201
202	if not (export_subsys.resolve or export_subsys.options.bin):	×
203	raise ExportError("Must specify at least one `--resolve` or `--bin` to export")	×
204	if targets:	×
205	raise ExportError("The `export` goal does not take target specs.")	×
206
207	requests = tuple(request_type(targets) for request_type in request_types)	×
208	all_results = await concurrently(	×
209	export(**implicitly({request: ExportRequest})) for request in requests
210	)
211	flattened_results = sorted(	×
212	(res for results in all_results for res in results), key=lambda res: res.resolve or ""
213	) # sorting provides predictable resolution in conflicts
214
215	prefixed_digests = await concurrently(	×
216	add_prefix(AddPrefix(result.digest, result.reldir)) for result in flattened_results
217	)
218	output_dir = os.path.join(str(dist_dir.relpath), "export")	×
219	for result in flattened_results:	×
220	digest_root = os.path.join(build_root.path, output_dir, result.reldir)	×
221	safe_rmtree(digest_root)	×
222	merged_digest = await merge_digests(MergeDigests(prefixed_digests))	×
223	dist_digest = await add_prefix(AddPrefix(merged_digest, output_dir))	×
224	workspace.write_digest(dist_digest)	×
225	environment = await environment_vars_subset(EnvironmentVarsRequest(["PATH"]), **implicitly())	×
226	resolves_exported = set()	×
227	for result in flattened_results:	×
228	result_dir = os.path.join(output_dir, result.reldir)	×
229	digest_root = os.path.join(build_root.path, result_dir)	×
230	for cmd in result.post_processing_cmds:	×
231	argv = tuple(arg.format(digest_root=digest_root) for arg in cmd.argv)	×
232	ip = InteractiveProcess(	×
233	argv=argv,
234	env={"PATH": environment.get("PATH", ""), **cmd.extra_env},
235	run_in_workspace=True,
236	)
237	ipr = await run_interactive_process(ip)	×
238	if ipr.exit_code:	×
239	raise ExportError(f"Failed to write {result.description} to {result_dir}")	×
240	if result.resolve:	×
241	resolves_exported.add(result.resolve)	×
242	console.print_stdout(f"Wrote {result.description} to {result_dir}")	×
243
244	exported_bins_by_exporting_resolve, link_requests = await link_exported_executables(	×
245	build_root, output_dir, flattened_results
246	)
247	link_digest = await create_digest(link_requests)	×
248	workspace.write_digest(link_digest)	×
249
250	exported_bin_warnings = warn_exported_bin_conflicts(exported_bins_by_exporting_resolve)	×
251	for warning in exported_bin_warnings:	×
252	console.print_stderr(warning)	×
253
254	unexported_resolves = sorted(	×
255	(set(export_subsys.resolve) \| set(export_subsys.binaries)) - resolves_exported
256	)
257	if unexported_resolves:	×
258	all_known_user_resolve_names = await concurrently(	×
259	get_known_user_resolve_names(**implicitly({request(): KnownUserResolveNamesRequest}))
260	for request in union_membership.get(KnownUserResolveNamesRequest)
261	)
262	all_known_bin_names = [	×
263	e.options_scope
264	for e in union_membership.get(ExportableTool)
265	if e.export_mode == ExportMode.binary
266	]
267	all_valid_resolve_names = sorted(	×
268	{
269	*itertools.chain.from_iterable(kurn.names for kurn in all_known_user_resolve_names),
270	}
271	)
272	raise UnrecognizedResolveNamesError(	×
273	unexported_resolves,
274	all_valid_resolve_names,
275	all_known_bin_names,
276	description_of_origin="the options --export-resolve and/or --export-bin",
277	)
278
279	return Export(exit_code=0)	×
280
281
282	async def link_exported_executables(	10✔
283	build_root: BuildRoot,
284	output_dir: str,
285	export_results: list[ExportResult],
286	) -> tuple[dict[str, list[str]], CreateDigest]:
287	"""Link the exported executables to the `bin` dir.
288
289	Multiple resolves might export the same executable. This will export the first one only but
290	track the collision.
291	"""
292	safe_mkdir(Path(output_dir, "bin"))	×
293
294	exported_bins_by_exporting_resolve: dict[str, list[str]] = defaultdict(list)	×
295	link_requests = []	×
296	for result in export_results:	×
297	for exported_bin in result.exported_binaries:	×
298	exported_bins_by_exporting_resolve[exported_bin.name].append(	×
299	result.resolve or result.description
300	)
301	if len(exported_bins_by_exporting_resolve[exported_bin.name]) > 1:	×
302	continue	×
303
304	path = Path(output_dir, "bin", exported_bin.name)	×
305	target = Path(build_root.path, output_dir, result.reldir, exported_bin.path_in_export)	×
306	link_requests.append(SymlinkEntry(path.as_posix(), target.as_posix()))	×
307
308	return exported_bins_by_exporting_resolve, CreateDigest(link_requests)	×
309
310
311	def warn_exported_bin_conflicts(exported_bins: dict[str, list[str]]) -> list[str]:	10✔
312	"""Check that no bin was exported from multiple resolves."""
313	messages = []	×
314
315	for exported_bin_name, resolves in exported_bins.items():	×
316	if len(resolves) > 1:	×
317	msg = f"Exporting binary `{exported_bin_name}` had conflicts. "	×
318	succeeded_resolve, other_resolves = resolves[0], resolves[1:]	×
319	msg += (	×
320	f"The resolve {succeeded_resolve} was exported, but it conflicted with "
321	+ ", ".join(other_resolves)
322	)
323	messages.append(msg)	×
324
325	return messages	×
326
327
328	def rules():	10✔
329	return collect_rules()	6✔

pantsbuild / pants / 18791134616

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous