• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 19383441979

15 Nov 2025 03:03AM UTC coverage: 80.284% (-0.006%) from 80.29%
19383441979

push

github

web-flow
plumb through (most) of Pex's --scie flags (#22866)

After playing with some back and forth I decided to add these to the
existing `pex_binary` target: that is most similar to what Pex does; and
it ended up being the easiest way to re-use existing machinery. I tried
to stay out of Pex's way with the various flags and pass things through.
The help text is lightly translated (ex: `-flags` too fields). The most
fiddly part is the expected outputs where the existing Pants code really
wants to know the expected outputs of a Process. I effectively
duplicated some of the Pex output logic, this is annoying, but I think
mostly getting in our own way instead of limiting what anyone can do,
and it does provide nicer output.

Flags not currently covered as fields:
* `--scie-only, --no-scie-only, --pex-and-scie`: The existing Pant
classes really expect to get a `.pex` out of all of this.
* `--scie-pypy-release`: I don't have any PyPy experience. I think it
would be straightforward to add.
* `--scie-pbs-free-threaded` / `--scie-pbs-debug`: This are new since I
started; thanks! Happy to do as a followup PR.
* `--scie-science-binary`: Would maybe be interesting if someone wrote a
general `science` backend.
 * `--scie-assets-base-url` / `--scie-base`

ref #22654

207 of 263 new or added lines in 5 files covered. (78.71%)

6 existing lines in 2 files now uncovered.

78075 of 97248 relevant lines covered (80.28%)

3.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

52.0
/src/python/pants/backend/python/goals/package_pex_binary.py
1
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
import dataclasses
10✔
5
import itertools
10✔
6
import logging
10✔
7
import os
10✔
8
from collections.abc import Iterable
10✔
9
from dataclasses import dataclass
10✔
10

11
from pants.backend.python.target_types import (
10✔
12
    PexArgsField,
13
    PexBinaryDefaults,
14
    PexCheckField,
15
    PexCompletePlatformsField,
16
    PexEmitWarningsField,
17
    PexEntryPointField,
18
    PexEnvField,
19
    PexExecutableField,
20
    PexExecutionMode,
21
    PexExecutionModeField,
22
    PexExtraBuildArgsField,
23
    PexIgnoreErrorsField,
24
    PexIncludeRequirementsField,
25
    PexIncludeSourcesField,
26
    PexIncludeToolsField,
27
    PexInheritPathField,
28
    PexLayout,
29
    PexLayoutField,
30
    PexScieBusyBox,
31
    PexScieBusyboxPexEntrypointEnvPassthrough,
32
    PexScieField,
33
    PexScieHashAlgField,
34
    PexScieNameStyleField,
35
    PexSciePbsReleaseField,
36
    PexSciePbsStripped,
37
    PexSciePlatformField,
38
    PexSciePythonVersion,
39
    PexScriptField,
40
    PexShBootField,
41
    PexShebangField,
42
    PexStripEnvField,
43
    PexVenvHermeticScripts,
44
    PexVenvSitePackagesCopies,
45
    ResolvePexEntryPointRequest,
46
    ScieNameStyle,
47
)
48
from pants.backend.python.target_types_rules import resolve_pex_entry_point
10✔
49
from pants.backend.python.util_rules.pex import create_pex, digest_complete_platforms
10✔
50
from pants.backend.python.util_rules.pex_from_targets import (
10✔
51
    PexFromTargetsRequest,
52
    create_pex_from_targets,
53
)
54
from pants.core.environments.target_types import EnvironmentField
10✔
55
from pants.core.goals.package import (
10✔
56
    BuiltPackage,
57
    BuiltPackageArtifact,
58
    OutputPathField,
59
    PackageFieldSet,
60
)
61
from pants.core.goals.run import RunFieldSet, RunInSandboxBehavior
10✔
62
from pants.engine.platform import Platform
10✔
63
from pants.engine.rules import collect_rules, implicitly, rule
10✔
64
from pants.engine.unions import UnionRule
10✔
65
from pants.util.frozendict import FrozenDict
10✔
66
from pants.util.logging import LogLevel
10✔
67

68
logger = logging.getLogger(__name__)
10✔
69

70

71
@dataclass(frozen=True)
10✔
72
class PexBinaryFieldSet(PackageFieldSet, RunFieldSet):
10✔
73
    run_in_sandbox_behavior = RunInSandboxBehavior.RUN_REQUEST_HERMETIC
10✔
74

75
    required_fields = (PexEntryPointField,)
10✔
76

77
    entry_point: PexEntryPointField
10✔
78
    script: PexScriptField
10✔
79
    executable: PexExecutableField
10✔
80
    args: PexArgsField
10✔
81
    env: PexEnvField
10✔
82

83
    output_path: OutputPathField
10✔
84
    emit_warnings: PexEmitWarningsField
10✔
85
    ignore_errors: PexIgnoreErrorsField
10✔
86
    inherit_path: PexInheritPathField
10✔
87
    sh_boot: PexShBootField
10✔
88
    shebang: PexShebangField
10✔
89
    strip_env: PexStripEnvField
10✔
90
    complete_platforms: PexCompletePlatformsField
10✔
91
    layout: PexLayoutField
10✔
92
    execution_mode: PexExecutionModeField
10✔
93
    include_requirements: PexIncludeRequirementsField
10✔
94
    include_sources: PexIncludeSourcesField
10✔
95
    include_tools: PexIncludeToolsField
10✔
96
    venv_site_packages_copies: PexVenvSitePackagesCopies
10✔
97
    venv_hermetic_scripts: PexVenvHermeticScripts
10✔
98
    environment: EnvironmentField
10✔
99
    check: PexCheckField
10✔
100
    extra_build_args: PexExtraBuildArgsField
10✔
101

102
    scie: PexScieField
10✔
103
    scie_name_style: PexScieNameStyleField
10✔
104
    scie_busybox: PexScieBusyBox
10✔
105
    scie_busybox_pex_entrypoint_env_passthrough: PexScieBusyboxPexEntrypointEnvPassthrough
10✔
106
    scie_platform: PexSciePlatformField
10✔
107
    scie_pbs_release: PexSciePbsReleaseField
10✔
108
    scie_python_version: PexSciePythonVersion
10✔
109
    scie_hash_alg: PexScieHashAlgField
10✔
110
    scie_pbs_stripped: PexSciePbsStripped
10✔
111

112
    def builds_pex_and_scie(self) -> bool:
10✔
NEW
113
        return self.scie.value is not None
×
114

115
    @property
10✔
116
    def _execution_mode(self) -> PexExecutionMode:
10✔
117
        return PexExecutionMode(self.execution_mode.value)
×
118

119
    def generate_additional_args(self, pex_binary_defaults: PexBinaryDefaults) -> tuple[str, ...]:
10✔
120
        args = []
×
121
        if self.emit_warnings.value_or_global_default(pex_binary_defaults) is False:
×
122
            args.append("--no-emit-warnings")
×
123
        elif self.emit_warnings.value_or_global_default(pex_binary_defaults) is True:
×
124
            args.append("--emit-warnings")
×
125
        if self.ignore_errors.value is True:
×
126
            args.append("--ignore-errors")
×
127
        if self.inherit_path.value is not None:
×
128
            args.append(f"--inherit-path={self.inherit_path.value}")
×
129
        if self.sh_boot.value is True:
×
130
            args.append("--sh-boot")
×
131
        if self.check.value is not None:
×
132
            args.append(f"--check={self.check.value}")
×
133
        if self.shebang.value is not None:
×
134
            args.append(f"--python-shebang={self.shebang.value}")
×
135
        if self.strip_env.value is False:
×
136
            args.append("--no-strip-pex-env")
×
137
        if self._execution_mode is PexExecutionMode.VENV:
×
138
            args.extend(("--venv", "prepend"))
×
139
        if self.include_tools.value is True:
×
140
            args.append("--include-tools")
×
141
        if self.venv_site_packages_copies.value is True:
×
142
            args.append("--venv-site-packages-copies")
×
143
        if self.venv_hermetic_scripts.value is False:
×
144
            args.append("--non-hermetic-venv-scripts")
×
145
        if self.extra_build_args.value:
×
146
            args.extend(self.extra_build_args.value)
×
147
        return tuple(args)
×
148

149
    def generate_scie_args(
10✔
150
        self,
151
    ) -> tuple[str, ...]:
NEW
152
        args = []
×
NEW
153
        if self.scie.value is not None:
×
NEW
154
            args.append(f"--scie={self.scie.value}")
×
NEW
155
        if self.scie_name_style.value is not None:
×
NEW
156
            args.append(f"--scie-name-style={self.scie_name_style.value}")
×
NEW
157
        if self.scie_busybox.value is not None:
×
NEW
158
            args.append(f"--scie-busybox={self.scie_busybox.value}")
×
NEW
159
        if self.scie_busybox_pex_entrypoint_env_passthrough.value is True:
×
NEW
160
            args.append("--scie-busybox-pex-entrypoint-env-passthrough")
×
NEW
161
        if self.scie_platform.value is not None:
×
NEW
162
            args.extend([f"--scie-platform={platform}" for platform in self.scie_platform.value])
×
NEW
163
        if self.scie_pbs_release.value is not None:
×
NEW
164
            args.append(f"--scie-pbs-release={self.scie_pbs_release.value}")
×
NEW
165
        if self.scie_python_version.value is not None:
×
NEW
166
            args.append(f"--scie-python-version={self.scie_python_version.value}")
×
NEW
167
        if self.scie_hash_alg.value is not None:
×
NEW
168
            args.append(f"--scie-hash-alg={self.scie_hash_alg.value}")
×
NEW
169
        if self.scie_pbs_stripped.value is True:
×
NEW
170
            args.append("--scie-pbs-stripped")
×
171

NEW
172
        return tuple(args)
×
173

174
    def output_pex_filename(self) -> str:
10✔
NEW
175
        return self.output_path.value_or_default(file_ending="pex")
×
176

177
    def scie_output_filenames(self) -> tuple[str, ...] | None:
10✔
NEW
178
        if not self.builds_pex_and_scie():
×
NEW
179
            return None
×
NEW
180
        return _scie_output_filenames(
×
181
            self.output_path.value_or_default(file_ending=None),
182
            self.scie_name_style.value
183
            if self.scie_name_style.value
184
            else self.scie_name_style.default,
185
            self.scie_platform.value,
186
            self.scie_hash_alg.value,
187
        )
188

189
    def scie_output_directories(self) -> tuple[str, ...] | None:
10✔
NEW
190
        if not self.builds_pex_and_scie():
×
NEW
191
            return None
×
NEW
192
        return _scie_output_directories(
×
193
            self.output_path.value_or_default(file_ending=None),
194
            self.scie_name_style.value
195
            if self.scie_name_style.value
196
            else self.scie_name_style.default,
197
            self.scie_platform.value,
198
        )
199

200

201
# Stand alone functions for ease of testing
202
def _current_scie_platform() -> str:
10✔
203
    # This is only a subset of the platforms that Pex can produce
204
    # scies for.  While Pants can produce foreign platform scies, the
205
    # "current" platform can only be one Pants itself can run on.
NEW
206
    platform = Platform.create_for_localhost().replace("_", "-")
×
NEW
207
    if platform == Platform.linux_arm64:
×
NEW
208
        return platform.replace("arm64", "aarch64")
×
209
    else:
NEW
210
        return platform
×
211

212

213
def _scie_output_filenames(
10✔
214
    no_suffix_output_path: str,
215
    scie_name_style: str,
216
    scie_platform: Iterable[str] | None,
217
    scie_hash_alg: str | None,
218
) -> tuple[str, ...] | None:
219
    filenames: list[str] = []
1✔
220

221
    if scie_name_style == ScieNameStyle.DYNAMIC:
1✔
222
        filenames = [no_suffix_output_path]
1✔
223
    elif scie_name_style == ScieNameStyle.PLATFORM_PARENT_DIR:
1✔
224
        return None  # handed by output_directories
1✔
225
    elif scie_name_style == ScieNameStyle.PLATFORM_FILE_SUFFIX:
1✔
226
        if scie_platform:
1✔
227
            filenames = [no_suffix_output_path + f"-{platform}" for platform in scie_platform]
1✔
228
        else:
NEW
229
            filenames = [no_suffix_output_path + f"-{_current_scie_platform()}"]
×
230

231
    if scie_hash_alg is None:
1✔
232
        return tuple(filenames)
1✔
233
    else:
234
        return tuple(
1✔
235
            itertools.chain.from_iterable(
236
                [(fname, f"{fname}.{scie_hash_alg}") for fname in filenames]
237
            )
238
        )
239

240

241
def _scie_output_directories(
10✔
242
    no_suffix_output_path: str,
243
    scie_name_style: str,
244
    scie_platform: Iterable[str] | None,
245
) -> tuple[str, ...] | None:
246
    if scie_name_style != ScieNameStyle.PLATFORM_PARENT_DIR:
1✔
247
        return None
1✔
248

249
    if scie_platform:
1✔
250
        return tuple(
1✔
251
            [
252
                os.path.join(os.path.dirname(no_suffix_output_path), platform)
253
                for platform in scie_platform
254
            ]
255
        )
256
    else:
NEW
257
        return tuple(os.path.join(os.path.dirname(no_suffix_output_path), _current_scie_platform()))
×
258

259

260
def _scie_build_package_artifacts(field_set: PexBinaryFieldSet) -> tuple[BuiltPackageArtifact, ...]:
10✔
NEW
261
    artifacts = []
×
262

NEW
263
    scie_output_filenames = field_set.scie_output_filenames()
×
NEW
264
    if scie_output_filenames is not None:
×
NEW
265
        artifacts.extend(
×
266
            [BuiltPackageArtifact(scie_filename) for scie_filename in scie_output_filenames]
267
        )
NEW
268
    scie_output_directories = field_set.scie_output_directories()
×
NEW
269
    if scie_output_directories is not None:
×
NEW
270
        artifacts.extend([BuiltPackageArtifact(scie_dir) for scie_dir in scie_output_directories])
×
NEW
271
    return tuple(artifacts)
×
272

273

274
@dataclass(frozen=True)
10✔
275
class PexFromTargetsRequestForBuiltPackage:
10✔
276
    """An intermediate class that gives consumers access to the data used to create a
277
    `PexFromTargetsRequest` to fulfil a `BuiltPackage` request.
278

279
    This class is used directly by `run_pex_binary`, but should be handled transparently by direct
280
    `BuiltPackage` requests.
281
    """
282

283
    request: PexFromTargetsRequest
10✔
284

285

286
@rule(level=LogLevel.DEBUG)
10✔
287
async def package_pex_binary(
10✔
288
    field_set: PexBinaryFieldSet,
289
    pex_binary_defaults: PexBinaryDefaults,
290
) -> PexFromTargetsRequestForBuiltPackage:
291
    resolved_entry_point = await resolve_pex_entry_point(
×
292
        ResolvePexEntryPointRequest(field_set.entry_point)
293
    )
294

NEW
295
    output_filename = field_set.output_pex_filename()
×
296

297
    complete_platforms = await digest_complete_platforms(field_set.complete_platforms)
×
298

299
    request = PexFromTargetsRequest(
×
300
        addresses=[field_set.address],
301
        internal_only=False,
302
        main=resolved_entry_point.val or field_set.script.value or field_set.executable.value,
303
        inject_args=field_set.args.value or [],
304
        inject_env=field_set.env.value or FrozenDict[str, str](),
305
        complete_platforms=complete_platforms,
306
        output_filename=output_filename,
307
        layout=PexLayout(field_set.layout.value),
308
        additional_args=field_set.generate_additional_args(pex_binary_defaults),
309
        include_requirements=field_set.include_requirements.value,
310
        include_source_files=field_set.include_sources.value,
311
        include_local_dists=True,
312
        warn_for_transitive_files_targets=True,
313
    )
314

315
    return PexFromTargetsRequestForBuiltPackage(request)
×
316

317

318
@rule
10✔
319
async def built_package_for_pex_from_targets_request(
10✔
320
    field_set: PexBinaryFieldSet,
321
) -> BuiltPackage:
322
    pft_request = await package_pex_binary(field_set, **implicitly())
×
323

NEW
324
    if field_set.builds_pex_and_scie():
×
NEW
325
        pex_request = dataclasses.replace(
×
326
            await create_pex_from_targets(**implicitly(pft_request.request)),
327
            additional_args=(*pft_request.request.additional_args, *field_set.generate_scie_args()),
328
            scie_output_files=field_set.scie_output_filenames(),
329
            scie_output_directories=field_set.scie_output_directories(),
330
        )
NEW
331
        artifacts = (
×
332
            BuiltPackageArtifact(
333
                pex_request.output_filename,
334
            ),
335
            *_scie_build_package_artifacts(field_set),
336
        )
337

338
    else:
NEW
339
        pex_request = await create_pex_from_targets(**implicitly(pft_request.request))
×
NEW
340
        artifacts = (
×
341
            BuiltPackageArtifact(
342
                pex_request.output_filename,
343
            ),
344
        )
345

NEW
346
    pex = await create_pex(**implicitly(pex_request))
×
347

NEW
348
    return BuiltPackage(pex.digest, artifacts)
×
349

350

351
def rules():
10✔
352
    return [*collect_rules(), UnionRule(PackageFieldSet, PexBinaryFieldSet)]
9✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc