• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 26342152999

23 May 2026 07:59PM UTC coverage: 91.165% (-1.6%) from 92.792%
26342152999

push

github

web-flow
Run Linux ARM CI on Depot runners (#23363)

RunsOn is deprecating their v2 stack, and rather than migrate
to v3 we should use the resources kindly donated by Depot.

GitHub also now has Linux ARM runners, should we need them.

87305 of 95766 relevant lines covered (91.16%)

3.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.26
/src/python/pants/backend/python/util_rules/dists.py
1
# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
from __future__ import annotations
8✔
5

6
import io
8✔
7
import os
8✔
8
from collections import abc
8✔
9
from collections.abc import Mapping
8✔
10
from dataclasses import dataclass
8✔
11
from typing import Any
8✔
12

13
import toml
8✔
14

15
from pants.backend.python.subsystems import setuptools
8✔
16
from pants.backend.python.subsystems.setup import PythonSetup
8✔
17
from pants.backend.python.subsystems.setuptools import Setuptools
8✔
18
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
8✔
19
from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPexProcess, create_venv_pex
8✔
20
from pants.backend.python.util_rules.pex import rules as pex_rules
8✔
21
from pants.backend.python.util_rules.pex_requirements import EntireLockfile, PexRequirements
8✔
22
from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
8✔
23
from pants.engine.fs import (
8✔
24
    CreateDigest,
25
    Digest,
26
    DigestSubset,
27
    FileContent,
28
    MergeDigests,
29
    PathGlobs,
30
    RemovePrefix,
31
)
32
from pants.engine.intrinsics import (
8✔
33
    create_digest,
34
    digest_to_snapshot,
35
    get_digest_contents,
36
    merge_digests,
37
    remove_prefix,
38
)
39
from pants.engine.process import fallible_to_exec_result_or_raise
8✔
40
from pants.engine.rules import collect_rules, implicitly, rule
8✔
41
from pants.util.frozendict import FrozenDict
8✔
42
from pants.util.logging import LogLevel
8✔
43
from pants.util.osutil import is_macos_big_sur
8✔
44
from pants.util.strutil import ensure_text, softwrap
8✔
45

46

47
class BuildBackendError(Exception):
8✔
48
    pass
8✔
49

50

51
class InvalidBuildConfigError(Exception):
8✔
52
    pass
8✔
53

54

55
@dataclass(frozen=True)
8✔
56
class BuildSystemRequest:
8✔
57
    """A request to find build system config in the given dir of the given digest."""
58

59
    digest: Digest
8✔
60
    working_directory: str
8✔
61

62

63
@dataclass(frozen=True)
8✔
64
class BuildSystem:
8✔
65
    """A PEP 517/518 build system configuration."""
66

67
    requires: PexRequirements | EntireLockfile
8✔
68
    build_backend: str
8✔
69

70
    @classmethod
8✔
71
    def legacy(cls, _setuptools: Setuptools) -> BuildSystem:
8✔
72
        return cls(_setuptools.pex_requirements(), "setuptools.build_meta:__legacy__")
3✔
73

74

75
@rule
8✔
76
async def find_build_system(request: BuildSystemRequest, _setuptools: Setuptools) -> BuildSystem:
8✔
77
    digest_contents = await get_digest_contents(
3✔
78
        **implicitly(
79
            DigestSubset(
80
                request.digest,
81
                PathGlobs(
82
                    globs=[os.path.join(request.working_directory, "pyproject.toml")],
83
                    glob_match_error_behavior=GlobMatchErrorBehavior.ignore,
84
                ),
85
            )
86
        )
87
    )
88
    ret = None
3✔
89
    if digest_contents:
3✔
90
        file_content = next(iter(digest_contents))
×
91
        settings: Mapping[str, Any] = toml.loads(file_content.content.decode())
×
92
        build_system = settings.get("build-system")
×
93
        if build_system is not None:
×
94
            build_backend = build_system.get("build-backend")
×
95
            if build_backend is None:
×
96
                raise InvalidBuildConfigError(
×
97
                    f"No build-backend found in the [build-system] table in {file_content.path}"
98
                )
99
            requires = build_system.get("requires")
×
100
            if requires is None:
×
101
                raise InvalidBuildConfigError(
×
102
                    f"No requires found in the [build-system] table in {file_content.path}"
103
                )
104
            ret = BuildSystem(PexRequirements(requires), build_backend)
×
105
    # Per PEP 517: "If the pyproject.toml file is absent, or the build-backend key is missing,
106
    #   the source tree is not using this specification, and tools should revert to the legacy
107
    #   behaviour of running setup.py."
108
    if ret is None:
3✔
109
        ret = BuildSystem.legacy(_setuptools)
3✔
110
    return ret
3✔
111

112

113
@dataclass(frozen=True)
8✔
114
class DistBuildRequest:
8✔
115
    """A request to build dists via a PEP 517 build backend."""
116

117
    build_system: BuildSystem
8✔
118

119
    # TODO: Support backend_path (https://www.python.org/dev/peps/pep-0517/#in-tree-build-backends)
120

121
    interpreter_constraints: InterpreterConstraints
8✔
122
    build_wheel: bool
8✔
123
    build_sdist: bool
8✔
124
    input: Digest
8✔
125
    working_directory: str  # Relpath within the input digest.
8✔
126
    dist_source_root: str  # Source root of the python_distribution target
8✔
127
    build_time_source_roots: tuple[str, ...]  # Source roots for 1st party build-time deps.
8✔
128
    output_path: str  # Location of the output directory within dist dir.
8✔
129

130
    target_address_spec: str | None = None  # Only needed for logging etc.
8✔
131
    wheel_config_settings: FrozenDict[str, tuple[str, ...]] | None = None
8✔
132
    sdist_config_settings: FrozenDict[str, tuple[str, ...]] | None = None
8✔
133

134
    extra_build_time_requirements: tuple[Pex, ...] = tuple()
8✔
135
    extra_build_time_env: Mapping[str, str] | None = None
8✔
136

137

138
@dataclass(frozen=True)
8✔
139
class DistBuildResult:
8✔
140
    output: Digest
8✔
141
    # Relpaths in the output digest.
142
    wheel_path: str | None
8✔
143
    sdist_path: str | None
8✔
144

145

146
# Note that the shim is capable of building a wheel and an sdist in one invocation, and that
147
# is how we currently run it.  We may in the future choose to invoke it twice, for finer-grained
148
# invalidation (e.g., so that we don't rebuild an sdist on wheel_config_settings changes).
149
# But then we would incur two process executions instead of one, and it's not yet clear if that is
150
# preferable. Even if we do decide to run in two passes, it'll still be better to have a single
151
# shim for both, so we can use the same merged digest, instead of needing two almost-identical
152
# ones, that differ only on the shim.
153
_BACKEND_SHIM_BOILERPLATE = """
8✔
154
# DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS
155

156
import errno
157
import os
158
import {build_backend_module}
159

160
backend = {build_backend_object}
161

162
dist_dir = "{dist_dir}"
163
build_wheel = {build_wheel}
164
build_sdist = {build_sdist}
165
wheel_config_settings = {wheel_config_settings_str}
166
sdist_config_settings = {sdist_config_settings_str}
167

168
# Python 2.7 doesn't have the exist_ok arg on os.makedirs().
169
try:
170
    os.makedirs(dist_dir)
171
except OSError as e:
172
    if e.errno != errno.EEXIST:
173
        raise
174

175
wheel_path = backend.build_wheel(dist_dir, wheel_config_settings) if build_wheel else None
176
sdist_path = backend.build_sdist(dist_dir, sdist_config_settings) if build_sdist else None
177

178
if wheel_path:
179
    print("wheel: {{wheel_path}}".format(wheel_path=wheel_path))
180
if sdist_path:
181
    print("sdist: {{sdist_path}}".format(sdist_path=sdist_path))
182
"""
183

184

185
def interpolate_backend_shim(dist_dir: str, request: DistBuildRequest) -> bytes:
8✔
186
    # See https://www.python.org/dev/peps/pep-0517/#source-trees.
187
    module_path, _, object_path = request.build_system.build_backend.partition(":")
2✔
188
    backend_object = f"{module_path}.{object_path}" if object_path else module_path
2✔
189

190
    def config_settings_repr(cs: FrozenDict[str, tuple[str, ...]] | None) -> str:
2✔
191
        # setuptools.build_meta expects list values and chokes on tuples.
192
        # We assume/hope that other backends accept lists as well.
193
        return distutils_repr(None if cs is None else {k: list(v) for k, v in cs.items()})
2✔
194

195
    return _BACKEND_SHIM_BOILERPLATE.format(
2✔
196
        build_backend_module=module_path,
197
        build_backend_object=backend_object,
198
        dist_dir=dist_dir,
199
        build_wheel=request.build_wheel,
200
        build_sdist=request.build_sdist,
201
        wheel_config_settings_str=config_settings_repr(request.wheel_config_settings),
202
        sdist_config_settings_str=config_settings_repr(request.sdist_config_settings),
203
    ).encode()
204

205

206
@rule
8✔
207
async def run_pep517_build(request: DistBuildRequest, python_setup: PythonSetup) -> DistBuildResult:
8✔
208
    # Note that this pex has no entrypoint. We use it to run our generated shim, which
209
    # in turn imports from and invokes the build backend.
210
    build_backend_pex = await create_venv_pex(
2✔
211
        **implicitly(
212
            PexRequest(
213
                output_filename="build_backend.pex",
214
                internal_only=True,
215
                requirements=request.build_system.requires,
216
                pex_path=request.extra_build_time_requirements,
217
                interpreter_constraints=request.interpreter_constraints,
218
            )
219
        )
220
    )
221

222
    # This is the setuptools dist directory, not Pants's, so we hardcode to dist/.
223
    dist_dir = "dist"
2✔
224
    backend_shim_name = "backend_shim.py"
2✔
225
    backend_shim_path = os.path.join(request.working_directory, backend_shim_name)
2✔
226
    backend_shim_digest = await create_digest(
2✔
227
        CreateDigest(
228
            [
229
                FileContent(
230
                    backend_shim_path,
231
                    interpolate_backend_shim(os.path.join(dist_dir, request.output_path), request),
232
                ),
233
            ]
234
        )
235
    )
236

237
    merged_digest = await merge_digests(MergeDigests((request.input, backend_shim_digest)))
2✔
238

239
    extra_env = {
2✔
240
        **(request.extra_build_time_env or {}),
241
        "PEX_EXTRA_SYS_PATH": os.pathsep.join(request.build_time_source_roots),
242
    }
243
    if python_setup.macos_big_sur_compatibility and is_macos_big_sur():
2✔
244
        extra_env["MACOSX_DEPLOYMENT_TARGET"] = "10.16"
×
245

246
    result = await fallible_to_exec_result_or_raise(
2✔
247
        **implicitly(
248
            VenvPexProcess(
249
                build_backend_pex,
250
                argv=(backend_shim_name,),
251
                input_digest=merged_digest,
252
                extra_env=extra_env,
253
                working_directory=request.working_directory,
254
                output_directories=(dist_dir,),  # Relative to the working_directory.
255
                description=(
256
                    f"Run {request.build_system.build_backend} for {request.target_address_spec}"
257
                    if request.target_address_spec
258
                    else f"Run {request.build_system.build_backend}"
259
                ),
260
                level=LogLevel.DEBUG,
261
            )
262
        )
263
    )
264
    output_lines = result.stdout.decode().splitlines()
2✔
265
    paths = {}
2✔
266
    for line in output_lines:
2✔
267
        for dist_type in ["wheel", "sdist"]:
2✔
268
            if line.startswith(f"{dist_type}: "):
2✔
269
                paths[dist_type] = os.path.join(
2✔
270
                    request.output_path, line[len(dist_type) + 2 :].strip()
271
                )
272
    # Note that output_digest paths are relative to the working_directory.
273
    output_digest = await remove_prefix(RemovePrefix(result.output_digest, dist_dir))
2✔
274
    output_snapshot = await digest_to_snapshot(output_digest)
2✔
275
    for dist_type, path in paths.items():
2✔
276
        if path not in output_snapshot.files:
2✔
277
            raise BuildBackendError(
×
278
                softwrap(
279
                    f"""
280
                    Build backend {request.build_system.build_backend} did not create
281
                    expected {dist_type} file {path}
282
                    """
283
                )
284
            )
285
    return DistBuildResult(
2✔
286
        output_digest, wheel_path=paths.get("wheel"), sdist_path=paths.get("sdist")
287
    )
288

289

290
# Distutils does not support unicode strings in setup.py, so we must explicitly convert to binary
291
# strings as pants uses unicode_literals. A natural and prior technique was to use `pprint.pformat`,
292
# but that embeds u's in the string itself during conversion. For that reason we roll out own
293
# literal pretty-printer here.
294
#
295
# Note that we must still keep this code, even though Pants only runs with Python 3, because
296
# the created product may still be run by Python 2.
297
#
298
# For more information, see http://bugs.python.org/issue13943.
299
def distutils_repr(obj) -> str:
8✔
300
    """Compute a string repr suitable for use in generated setup.py files."""
301
    output = io.StringIO()
4✔
302
    linesep = os.linesep
4✔
303

304
    def _write(data):
4✔
305
        output.write(ensure_text(data))
4✔
306

307
    def _write_repr(o, indent=False, level=0):
4✔
308
        pad = " " * 4 * level
4✔
309
        if indent:
4✔
310
            _write(pad)
3✔
311
        level += 1
4✔
312

313
        if isinstance(o, (bytes, str)):
4✔
314
            # The py2 repr of str (unicode) is `u'...'` and we don't want the `u` prefix; likewise,
315
            # the py3 repr of bytes is `b'...'` and we don't want the `b` prefix so we hand-roll a
316
            # repr here.
317
            o_txt = ensure_text(o)
3✔
318
            if linesep in o_txt:
3✔
319
                _write('"""{}"""'.format(o_txt.replace('"""', r"\"\"\"")))
×
320
            else:
321
                _write("'{}'".format(o_txt.replace("'", r"\'")))
3✔
322
        elif isinstance(o, abc.Mapping):
4✔
323
            _write("{" + linesep)
4✔
324
            for k, v in o.items():
4✔
325
                _write_repr(k, indent=True, level=level)
3✔
326
                _write(": ")
3✔
327
                _write_repr(v, indent=False, level=level)
3✔
328
                _write("," + linesep)
3✔
329
            _write(pad + "}")
4✔
330
        elif isinstance(o, abc.Iterable):
3✔
331
            if isinstance(o, abc.MutableSequence):
3✔
332
                open_collection, close_collection = "[]"
1✔
333
            elif isinstance(o, abc.Set):
3✔
334
                open_collection, close_collection = "{}"
×
335
            else:
336
                open_collection, close_collection = "()"
3✔
337

338
            _write(open_collection + linesep)
3✔
339
            for i in o:
3✔
340
                _write_repr(i, indent=True, level=level)
3✔
341
                _write("," + linesep)
3✔
342
            _write(pad + close_collection)
3✔
343
        else:
344
            _write(repr(o))  # Numbers and bools.
×
345

346
    _write_repr(obj)
4✔
347
    return output.getvalue()
4✔
348

349

350
def rules():
8✔
351
    return (*collect_rules(), *setuptools.rules(), *pex_rules())
8✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc