• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 20332790708

18 Dec 2025 09:48AM UTC coverage: 64.992% (-15.3%) from 80.295%
20332790708

Pull #22949

github

web-flow
Merge f730a56cd into 407284c67
Pull Request #22949: Add experimental uv resolver for Python lockfiles

54 of 97 new or added lines in 5 files covered. (55.67%)

8270 existing lines in 295 files now uncovered.

48990 of 75379 relevant lines covered (64.99%)

1.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

53.85
/src/python/pants/core/goals/export.py
1
# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
from __future__ import annotations
5✔
5

6
import itertools
5✔
7
import os
5✔
8
from collections import defaultdict
5✔
9
from collections.abc import Iterable, Mapping, Sequence
5✔
10
from dataclasses import dataclass
5✔
11
from pathlib import Path
5✔
12
from typing import cast
5✔
13

14
from pants.base.build_root import BuildRoot
5✔
15
from pants.core.goals.generate_lockfiles import (
5✔
16
    KnownUserResolveNamesRequest,
17
    UnrecognizedResolveNamesError,
18
    get_known_user_resolve_names,
19
)
20
from pants.core.goals.resolves import ExportableTool, ExportMode
5✔
21
from pants.core.util_rules.distdir import DistDir
5✔
22
from pants.core.util_rules.env_vars import environment_vars_subset
5✔
23
from pants.engine.collection import Collection
5✔
24
from pants.engine.console import Console
5✔
25
from pants.engine.env_vars import EnvironmentVarsRequest
5✔
26
from pants.engine.environment import EnvironmentName
5✔
27
from pants.engine.fs import (
5✔
28
    EMPTY_DIGEST,
29
    AddPrefix,
30
    CreateDigest,
31
    Digest,
32
    MergeDigests,
33
    SymlinkEntry,
34
    Workspace,
35
)
36
from pants.engine.goal import Goal, GoalSubsystem
5✔
37
from pants.engine.internals.selectors import concurrently
5✔
38
from pants.engine.intrinsics import (
5✔
39
    add_prefix,
40
    create_digest,
41
    merge_digests,
42
    run_interactive_process,
43
)
44
from pants.engine.process import InteractiveProcess
5✔
45
from pants.engine.rules import collect_rules, goal_rule, implicitly, rule
5✔
46
from pants.engine.target import FilteredTargets, Target
5✔
47
from pants.engine.unions import UnionMembership, union
5✔
48
from pants.option.option_types import StrListOption
5✔
49
from pants.util.dirutil import safe_mkdir, safe_rmtree
5✔
50
from pants.util.frozendict import FrozenDict
5✔
51
from pants.util.strutil import softwrap
5✔
52

53

54
class ExportError(Exception):
5✔
55
    pass
5✔
56

57

58
@union(in_scope_types=[EnvironmentName])
5✔
59
@dataclass(frozen=True)
5✔
60
class ExportRequest:
5✔
61
    """A union for exportable data provided by a backend.
62

63
    Subclass and install a member of this type to export data.
64
    """
65

66
    targets: Sequence[Target]
5✔
67

68

69
@dataclass(frozen=True)
5✔
70
class PostProcessingCommand:
5✔
71
    """A command to run as a local process after an exported digest is materialized."""
72

73
    # Values in the argv tuple can contain the format specifier "{digest_root}", which will be
74
    # substituted with the (absolute) path to the location under distdir in which the
75
    # digest is materialized.
76
    argv: tuple[str, ...]
5✔
77
    # The command will be run with an environment consisting of just PATH, set to the Pants
78
    # process's own PATH env var, plus these extra env vars.
79
    extra_env: FrozenDict[str, str]
5✔
80

81
    def __init__(
5✔
82
        self,
83
        argv: Iterable[str],
84
        extra_env: Mapping[str, str] = FrozenDict(),
85
    ):
UNCOV
86
        object.__setattr__(self, "argv", tuple(argv))
×
UNCOV
87
        object.__setattr__(self, "extra_env", FrozenDict(extra_env))
×
88

89

90
@dataclass(frozen=True)
5✔
91
class ExportedBinary:
5✔
92
    """Binaries exposed by an export.
93

94
    These will be added under the "bin" folder. The `name` is the name that will be linked as in the
95
    `bin` folder. The `path_in_export` is the path within the exported digest to link to. These can
96
    be used to abstract details from the name of the tool and avoid the other files in the tool's
97
    digest.
98

99
    For example, "my-tool" might have a downloaded file of
100
    "my_tool/my_tool_linux_x86-64.bin" and a readme. We would use `ExportedBinary(name="my-tool",
101
    path_in_export=my_tool/my_tool_linux_x86-64.bin"`
102
    """
103

104
    name: str
5✔
105
    path_in_export: str
5✔
106

107

108
@dataclass(frozen=True)
5✔
109
class ExportResult:
5✔
110
    description: str
5✔
111
    # Materialize digests under this reldir.
112
    reldir: str
5✔
113
    # Materialize this digest.
114
    digest: Digest
5✔
115
    # Run these commands as local processes after the digest is materialized.
116
    post_processing_cmds: tuple[PostProcessingCommand, ...]
5✔
117
    # Set for the common special case of exporting a resolve, and names that resolve.
118
    # Set to None for other export results.
119
    resolve: str | None
5✔
120
    exported_binaries: tuple[ExportedBinary, ...]
5✔
121

122
    def __init__(
5✔
123
        self,
124
        description: str,
125
        reldir: str,
126
        *,
127
        digest: Digest = EMPTY_DIGEST,
128
        post_processing_cmds: Iterable[PostProcessingCommand] = tuple(),
129
        resolve: str | None = None,
130
        exported_binaries: Iterable[ExportedBinary] = tuple(),
131
    ):
132
        object.__setattr__(self, "description", description)
1✔
133
        object.__setattr__(self, "reldir", reldir)
1✔
134
        object.__setattr__(self, "digest", digest)
1✔
135
        object.__setattr__(self, "post_processing_cmds", tuple(post_processing_cmds))
1✔
136
        object.__setattr__(self, "resolve", resolve)
1✔
137
        object.__setattr__(self, "exported_binaries", tuple(exported_binaries))
1✔
138

139

140
class ExportResults(Collection[ExportResult]):
5✔
141
    pass
5✔
142

143

144
@rule(polymorphic=True)
5✔
145
async def export(
5✔
146
    req: ExportRequest,
147
    environment_name: EnvironmentName,
148
) -> ExportResults:
149
    raise NotImplementedError()
×
150

151

152
class ExportSubsystem(GoalSubsystem):
5✔
153
    name = "export"
5✔
154
    help = softwrap(
5✔
155
        """
156
        Export Pants data for use in other tools, such as IDEs.
157

158
        :::caution Exporting tools requires creating a custom lockfile for them
159

160
        Follow [the instructions for creating tool lockfiles](../../docs/python/overview/lockfiles#lockfiles-for-tools)
161

162
        :::
163
        """
164
    )
165

166
    # NB: Only options that are relevant across many/most backends and languages
167
    #  should be defined here.  Backend-specific options should be defined in that backend
168
    #  as plugin options on this subsystem.
169

170
    # Exporting resolves is a common use-case for `export`, often the primary one, so we
171
    # add affordances for it at the core goal level.
172
    resolve = StrListOption(
5✔
173
        default=[],
174
        help="Export the specified resolve(s). The export format is backend-specific, "
175
        "e.g., Python resolves are exported as virtualenvs.",
176
    )
177

178
    binaries = StrListOption(
5✔
179
        flag_name="--bin",  # `bin` is a python builtin
180
        default=[],
181
        help="Export the specified binaries. To select a binary, provide its subsystem scope name, as used for setting its options.",
182
    )
183

184

185
class Export(Goal):
5✔
186
    subsystem_cls = ExportSubsystem
5✔
187
    environment_behavior = Goal.EnvironmentBehavior.LOCAL_ONLY
5✔
188

189

190
@goal_rule
5✔
191
async def export_goal(
5✔
192
    console: Console,
193
    targets: FilteredTargets,
194
    workspace: Workspace,
195
    union_membership: UnionMembership,
196
    build_root: BuildRoot,
197
    dist_dir: DistDir,
198
    export_subsys: ExportSubsystem,
199
) -> Export:
UNCOV
200
    request_types = cast("Iterable[type[ExportRequest]]", union_membership.get(ExportRequest))
×
201

UNCOV
202
    if not (export_subsys.resolve or export_subsys.options.bin):
×
203
        raise ExportError("Must specify at least one `--resolve` or `--bin` to export")
×
UNCOV
204
    if targets:
×
205
        raise ExportError("The `export` goal does not take target specs.")
×
206

UNCOV
207
    requests = tuple(request_type(targets) for request_type in request_types)
×
UNCOV
208
    all_results = await concurrently(
×
209
        export(**implicitly({request: ExportRequest})) for request in requests
210
    )
UNCOV
211
    flattened_results = sorted(
×
212
        (res for results in all_results for res in results), key=lambda res: res.resolve or ""
213
    )  # sorting provides predictable resolution in conflicts
214

UNCOV
215
    prefixed_digests = await concurrently(
×
216
        add_prefix(AddPrefix(result.digest, result.reldir)) for result in flattened_results
217
    )
UNCOV
218
    output_dir = os.path.join(str(dist_dir.relpath), "export")
×
UNCOV
219
    for result in flattened_results:
×
UNCOV
220
        digest_root = os.path.join(build_root.path, output_dir, result.reldir)
×
UNCOV
221
        safe_rmtree(digest_root)
×
UNCOV
222
    merged_digest = await merge_digests(MergeDigests(prefixed_digests))
×
UNCOV
223
    dist_digest = await add_prefix(AddPrefix(merged_digest, output_dir))
×
UNCOV
224
    workspace.write_digest(dist_digest)
×
UNCOV
225
    environment = await environment_vars_subset(EnvironmentVarsRequest(["PATH"]), **implicitly())
×
UNCOV
226
    resolves_exported = set()
×
UNCOV
227
    for result in flattened_results:
×
UNCOV
228
        result_dir = os.path.join(output_dir, result.reldir)
×
UNCOV
229
        digest_root = os.path.join(build_root.path, result_dir)
×
UNCOV
230
        for cmd in result.post_processing_cmds:
×
UNCOV
231
            argv = tuple(arg.format(digest_root=digest_root) for arg in cmd.argv)
×
UNCOV
232
            ip = InteractiveProcess(
×
233
                argv=argv,
234
                env={"PATH": environment.get("PATH", ""), **cmd.extra_env},
235
                run_in_workspace=True,
236
            )
UNCOV
237
            ipr = await run_interactive_process(ip)
×
UNCOV
238
            if ipr.exit_code:
×
239
                raise ExportError(f"Failed to write {result.description} to {result_dir}")
×
UNCOV
240
        if result.resolve:
×
UNCOV
241
            resolves_exported.add(result.resolve)
×
UNCOV
242
        console.print_stdout(f"Wrote {result.description} to {result_dir}")
×
243

UNCOV
244
    exported_bins_by_exporting_resolve, link_requests = await link_exported_executables(
×
245
        build_root, output_dir, flattened_results
246
    )
UNCOV
247
    link_digest = await create_digest(link_requests)
×
UNCOV
248
    workspace.write_digest(link_digest)
×
249

UNCOV
250
    exported_bin_warnings = warn_exported_bin_conflicts(exported_bins_by_exporting_resolve)
×
UNCOV
251
    for warning in exported_bin_warnings:
×
252
        console.print_stderr(warning)
×
253

UNCOV
254
    unexported_resolves = sorted(
×
255
        (set(export_subsys.resolve) | set(export_subsys.binaries)) - resolves_exported
256
    )
UNCOV
257
    if unexported_resolves:
×
258
        all_known_user_resolve_names = await concurrently(
×
259
            get_known_user_resolve_names(**implicitly({request(): KnownUserResolveNamesRequest}))
260
            for request in union_membership.get(KnownUserResolveNamesRequest)
261
        )
262
        all_known_bin_names = [
×
263
            e.options_scope
264
            for e in union_membership.get(ExportableTool)
265
            if e.export_mode == ExportMode.binary
266
        ]
267
        all_valid_resolve_names = sorted(
×
268
            {
269
                *itertools.chain.from_iterable(kurn.names for kurn in all_known_user_resolve_names),
270
            }
271
        )
272
        raise UnrecognizedResolveNamesError(
×
273
            unexported_resolves,
274
            all_valid_resolve_names,
275
            all_known_bin_names,
276
            description_of_origin="the options --export-resolve and/or --export-bin",
277
        )
278

UNCOV
279
    return Export(exit_code=0)
×
280

281

282
async def link_exported_executables(
5✔
283
    build_root: BuildRoot,
284
    output_dir: str,
285
    export_results: list[ExportResult],
286
) -> tuple[dict[str, list[str]], CreateDigest]:
287
    """Link the exported executables to the `bin` dir.
288

289
    Multiple resolves might export the same executable. This will export the first one only but
290
    track the collision.
291
    """
UNCOV
292
    safe_mkdir(Path(output_dir, "bin"))
×
293

UNCOV
294
    exported_bins_by_exporting_resolve: dict[str, list[str]] = defaultdict(list)
×
UNCOV
295
    link_requests = []
×
UNCOV
296
    for result in export_results:
×
UNCOV
297
        for exported_bin in result.exported_binaries:
×
298
            exported_bins_by_exporting_resolve[exported_bin.name].append(
×
299
                result.resolve or result.description
300
            )
301
            if len(exported_bins_by_exporting_resolve[exported_bin.name]) > 1:
×
302
                continue
×
303

304
            path = Path(output_dir, "bin", exported_bin.name)
×
305
            target = Path(build_root.path, output_dir, result.reldir, exported_bin.path_in_export)
×
306
            link_requests.append(SymlinkEntry(path.as_posix(), target.as_posix()))
×
307

UNCOV
308
    return exported_bins_by_exporting_resolve, CreateDigest(link_requests)
×
309

310

311
def warn_exported_bin_conflicts(exported_bins: dict[str, list[str]]) -> list[str]:
5✔
312
    """Check that no bin was exported from multiple resolves."""
UNCOV
313
    messages = []
×
314

UNCOV
315
    for exported_bin_name, resolves in exported_bins.items():
×
UNCOV
316
        if len(resolves) > 1:
×
UNCOV
317
            msg = f"Exporting binary `{exported_bin_name}` had conflicts. "
×
UNCOV
318
            succeeded_resolve, other_resolves = resolves[0], resolves[1:]
×
UNCOV
319
            msg += (
×
320
                f"The resolve {succeeded_resolve} was exported, but it conflicted with "
321
                + ", ".join(other_resolves)
322
            )
UNCOV
323
            messages.append(msg)
×
324

UNCOV
325
    return messages
×
326

327

328
def rules():
5✔
329
    return collect_rules()
2✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc