• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 18252174847

05 Oct 2025 01:36AM UTC coverage: 43.382% (-36.9%) from 80.261%
18252174847

push

github

web-flow
run tests on mac arm (#22717)

Just doing the minimal to pull forward the x86_64 pattern.

ref #20993

25776 of 59416 relevant lines covered (43.38%)

1.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

45.64
/src/python/pants/backend/python/util_rules/dists.py
1
# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
from __future__ import annotations
3✔
5

6
import io
3✔
7
import os
3✔
8
from collections import abc
3✔
9
from collections.abc import Mapping
3✔
10
from dataclasses import dataclass
3✔
11
from typing import Any
3✔
12

13
import toml
3✔
14

15
from pants.backend.python.subsystems import setuptools
3✔
16
from pants.backend.python.subsystems.setup import PythonSetup
3✔
17
from pants.backend.python.subsystems.setuptools import Setuptools
3✔
18
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
3✔
19
from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPexProcess, create_venv_pex
3✔
20
from pants.backend.python.util_rules.pex import rules as pex_rules
3✔
21
from pants.backend.python.util_rules.pex_requirements import EntireLockfile, PexRequirements
3✔
22
from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
3✔
23
from pants.engine.fs import (
3✔
24
    CreateDigest,
25
    Digest,
26
    DigestSubset,
27
    FileContent,
28
    MergeDigests,
29
    PathGlobs,
30
    RemovePrefix,
31
)
32
from pants.engine.intrinsics import (
3✔
33
    create_digest,
34
    digest_to_snapshot,
35
    get_digest_contents,
36
    merge_digests,
37
    remove_prefix,
38
)
39
from pants.engine.process import fallible_to_exec_result_or_raise
3✔
40
from pants.engine.rules import collect_rules, implicitly, rule
3✔
41
from pants.util.frozendict import FrozenDict
3✔
42
from pants.util.logging import LogLevel
3✔
43
from pants.util.osutil import is_macos_big_sur
3✔
44
from pants.util.strutil import ensure_text, softwrap
3✔
45

46

47
class BuildBackendError(Exception):
3✔
48
    pass
3✔
49

50

51
class InvalidBuildConfigError(Exception):
3✔
52
    pass
3✔
53

54

55
@dataclass(frozen=True)
3✔
56
class BuildSystemRequest:
3✔
57
    """A request to find build system config in the given dir of the given digest."""
58

59
    digest: Digest
3✔
60
    working_directory: str
3✔
61

62

63
@dataclass(frozen=True)
3✔
64
class BuildSystem:
3✔
65
    """A PEP 517/518 build system configuration."""
66

67
    requires: PexRequirements | EntireLockfile
3✔
68
    build_backend: str
3✔
69

70
    @classmethod
3✔
71
    def legacy(cls, _setuptools: Setuptools) -> BuildSystem:
3✔
72
        return cls(_setuptools.pex_requirements(), "setuptools.build_meta:__legacy__")
×
73

74

75
@rule
3✔
76
async def find_build_system(request: BuildSystemRequest, _setuptools: Setuptools) -> BuildSystem:
3✔
77
    digest_contents = await get_digest_contents(
×
78
        **implicitly(
79
            DigestSubset(
80
                request.digest,
81
                PathGlobs(
82
                    globs=[os.path.join(request.working_directory, "pyproject.toml")],
83
                    glob_match_error_behavior=GlobMatchErrorBehavior.ignore,
84
                ),
85
            )
86
        )
87
    )
88
    ret = None
×
89
    if digest_contents:
×
90
        file_content = next(iter(digest_contents))
×
91
        settings: Mapping[str, Any] = toml.loads(file_content.content.decode())
×
92
        build_system = settings.get("build-system")
×
93
        if build_system is not None:
×
94
            build_backend = build_system.get("build-backend")
×
95
            if build_backend is None:
×
96
                raise InvalidBuildConfigError(
×
97
                    f"No build-backend found in the [build-system] table in {file_content.path}"
98
                )
99
            requires = build_system.get("requires")
×
100
            if requires is None:
×
101
                raise InvalidBuildConfigError(
×
102
                    f"No requires found in the [build-system] table in {file_content.path}"
103
                )
104
            ret = BuildSystem(PexRequirements(requires), build_backend)
×
105
    # Per PEP 517: "If the pyproject.toml file is absent, or the build-backend key is missing,
106
    #   the source tree is not using this specification, and tools should revert to the legacy
107
    #   behaviour of running setup.py."
108
    if ret is None:
×
109
        ret = BuildSystem.legacy(_setuptools)
×
110
    return ret
×
111

112

113
@dataclass(frozen=True)
3✔
114
class DistBuildRequest:
3✔
115
    """A request to build dists via a PEP 517 build backend."""
116

117
    build_system: BuildSystem
3✔
118

119
    # TODO: Support backend_path (https://www.python.org/dev/peps/pep-0517/#in-tree-build-backends)
120

121
    interpreter_constraints: InterpreterConstraints
3✔
122
    build_wheel: bool
3✔
123
    build_sdist: bool
3✔
124
    input: Digest
3✔
125
    working_directory: str  # Relpath within the input digest.
3✔
126
    dist_source_root: str  # Source root of the python_distribution target
3✔
127
    build_time_source_roots: tuple[str, ...]  # Source roots for 1st party build-time deps.
3✔
128
    output_path: str  # Location of the output directory within dist dir.
3✔
129

130
    target_address_spec: str | None = None  # Only needed for logging etc.
3✔
131
    wheel_config_settings: FrozenDict[str, tuple[str, ...]] | None = None
3✔
132
    sdist_config_settings: FrozenDict[str, tuple[str, ...]] | None = None
3✔
133

134
    extra_build_time_requirements: tuple[Pex, ...] = tuple()
3✔
135
    extra_build_time_env: Mapping[str, str] | None = None
3✔
136

137

138
@dataclass(frozen=True)
3✔
139
class DistBuildResult:
3✔
140
    output: Digest
3✔
141
    # Relpaths in the output digest.
142
    wheel_path: str | None
3✔
143
    sdist_path: str | None
3✔
144

145

146
# Note that the shim is capable of building a wheel and an sdist in one invocation, and that
147
# is how we currently run it.  We may in the future choose to invoke it twice, for finer-grained
148
# invalidation (e.g., so that we don't rebuild an sdist on wheel_config_settings changes).
149
# But then we would incur two process executions instead of one, and it's not yet clear if that is
150
# preferable. Even if we do decide to run in two passes, it'll still be better to have a single
151
# shim for both, so we can use the same merged digest, instead of needing two almost-identical
152
# ones, that differ only on the shim.
153
_BACKEND_SHIM_BOILERPLATE = """
3✔
154
# DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS
155

156
import errno
157
import os
158
import {build_backend_module}
159

160
backend = {build_backend_object}
161

162
dist_dir = "{dist_dir}"
163
build_wheel = {build_wheel}
164
build_sdist = {build_sdist}
165
wheel_config_settings = {wheel_config_settings_str}
166
sdist_config_settings = {sdist_config_settings_str}
167

168
# Python 2.7 doesn't have the exist_ok arg on os.makedirs().
169
try:
170
    os.makedirs(dist_dir)
171
except OSError as e:
172
    if e.errno != errno.EEXIST:
173
        raise
174

175
wheel_path = backend.build_wheel(dist_dir, wheel_config_settings) if build_wheel else None
176
sdist_path = backend.build_sdist(dist_dir, sdist_config_settings) if build_sdist else None
177

178
if wheel_path:
179
    print("wheel: {{wheel_path}}".format(wheel_path=wheel_path))
180
if sdist_path:
181
    print("sdist: {{sdist_path}}".format(sdist_path=sdist_path))
182
"""
183

184

185
def interpolate_backend_shim(dist_dir: str, request: DistBuildRequest) -> bytes:
3✔
186
    # See https://www.python.org/dev/peps/pep-0517/#source-trees.
187
    module_path, _, object_path = request.build_system.build_backend.partition(":")
×
188
    backend_object = f"{module_path}.{object_path}" if object_path else module_path
×
189

190
    def config_settings_repr(cs: FrozenDict[str, tuple[str, ...]] | None) -> str:
×
191
        # setuptools.build_meta expects list values and chokes on tuples.
192
        # We assume/hope that other backends accept lists as well.
193
        return distutils_repr(None if cs is None else {k: list(v) for k, v in cs.items()})
×
194

195
    return _BACKEND_SHIM_BOILERPLATE.format(
×
196
        build_backend_module=module_path,
197
        build_backend_object=backend_object,
198
        dist_dir=dist_dir,
199
        build_wheel=request.build_wheel,
200
        build_sdist=request.build_sdist,
201
        wheel_config_settings_str=config_settings_repr(request.wheel_config_settings),
202
        sdist_config_settings_str=config_settings_repr(request.sdist_config_settings),
203
    ).encode()
204

205

206
@rule
3✔
207
async def run_pep517_build(request: DistBuildRequest, python_setup: PythonSetup) -> DistBuildResult:
3✔
208
    # Note that this pex has no entrypoint. We use it to run our generated shim, which
209
    # in turn imports from and invokes the build backend.
210
    build_backend_pex = await create_venv_pex(
×
211
        **implicitly(
212
            PexRequest(
213
                output_filename="build_backend.pex",
214
                internal_only=True,
215
                requirements=request.build_system.requires,
216
                pex_path=request.extra_build_time_requirements,
217
                interpreter_constraints=request.interpreter_constraints,
218
            )
219
        )
220
    )
221

222
    # This is the setuptools dist directory, not Pants's, so we hardcode to dist/.
223
    dist_dir = "dist"
×
224
    backend_shim_name = "backend_shim.py"
×
225
    backend_shim_path = os.path.join(request.working_directory, backend_shim_name)
×
226
    backend_shim_digest = await create_digest(
×
227
        CreateDigest(
228
            [
229
                FileContent(
230
                    backend_shim_path,
231
                    interpolate_backend_shim(os.path.join(dist_dir, request.output_path), request),
232
                ),
233
            ]
234
        )
235
    )
236

237
    merged_digest = await merge_digests(MergeDigests((request.input, backend_shim_digest)))
×
238

239
    extra_env = {
×
240
        **(request.extra_build_time_env or {}),
241
        "PEX_EXTRA_SYS_PATH": os.pathsep.join(request.build_time_source_roots),
242
    }
243
    if python_setup.macos_big_sur_compatibility and is_macos_big_sur():
×
244
        extra_env["MACOSX_DEPLOYMENT_TARGET"] = "10.16"
×
245

246
    result = await fallible_to_exec_result_or_raise(
×
247
        **implicitly(
248
            VenvPexProcess(
249
                build_backend_pex,
250
                argv=(backend_shim_name,),
251
                input_digest=merged_digest,
252
                extra_env=extra_env,
253
                working_directory=request.working_directory,
254
                output_directories=(dist_dir,),  # Relative to the working_directory.
255
                description=(
256
                    f"Run {request.build_system.build_backend} for {request.target_address_spec}"
257
                    if request.target_address_spec
258
                    else f"Run {request.build_system.build_backend}"
259
                ),
260
                level=LogLevel.DEBUG,
261
            )
262
        )
263
    )
264
    output_lines = result.stdout.decode().splitlines()
×
265
    paths = {}
×
266
    for line in output_lines:
×
267
        for dist_type in ["wheel", "sdist"]:
×
268
            if line.startswith(f"{dist_type}: "):
×
269
                paths[dist_type] = os.path.join(
×
270
                    request.output_path, line[len(dist_type) + 2 :].strip()
271
                )
272
    # Note that output_digest paths are relative to the working_directory.
273
    output_digest = await remove_prefix(RemovePrefix(result.output_digest, dist_dir))
×
274
    output_snapshot = await digest_to_snapshot(output_digest)
×
275
    for dist_type, path in paths.items():
×
276
        if path not in output_snapshot.files:
×
277
            raise BuildBackendError(
×
278
                softwrap(
279
                    f"""
280
                    Build backend {request.build_system.build_backend} did not create
281
                    expected {dist_type} file {path}
282
                    """
283
                )
284
            )
285
    return DistBuildResult(
×
286
        output_digest, wheel_path=paths.get("wheel"), sdist_path=paths.get("sdist")
287
    )
288

289

290
# Distutils does not support unicode strings in setup.py, so we must explicitly convert to binary
291
# strings as pants uses unicode_literals. A natural and prior technique was to use `pprint.pformat`,
292
# but that embeds u's in the string itself during conversion. For that reason we roll out own
293
# literal pretty-printer here.
294
#
295
# Note that we must still keep this code, even though Pants only runs with Python 3, because
296
# the created product may still be run by Python 2.
297
#
298
# For more information, see http://bugs.python.org/issue13943.
299
def distutils_repr(obj) -> str:
3✔
300
    """Compute a string repr suitable for use in generated setup.py files."""
301
    output = io.StringIO()
×
302
    linesep = os.linesep
×
303

304
    def _write(data):
×
305
        output.write(ensure_text(data))
×
306

307
    def _write_repr(o, indent=False, level=0):
×
308
        pad = " " * 4 * level
×
309
        if indent:
×
310
            _write(pad)
×
311
        level += 1
×
312

313
        if isinstance(o, (bytes, str)):
×
314
            # The py2 repr of str (unicode) is `u'...'` and we don't want the `u` prefix; likewise,
315
            # the py3 repr of bytes is `b'...'` and we don't want the `b` prefix so we hand-roll a
316
            # repr here.
317
            o_txt = ensure_text(o)
×
318
            if linesep in o_txt:
×
319
                _write('"""{}"""'.format(o_txt.replace('"""', r"\"\"\"")))
×
320
            else:
321
                _write("'{}'".format(o_txt.replace("'", r"\'")))
×
322
        elif isinstance(o, abc.Mapping):
×
323
            _write("{" + linesep)
×
324
            for k, v in o.items():
×
325
                _write_repr(k, indent=True, level=level)
×
326
                _write(": ")
×
327
                _write_repr(v, indent=False, level=level)
×
328
                _write("," + linesep)
×
329
            _write(pad + "}")
×
330
        elif isinstance(o, abc.Iterable):
×
331
            if isinstance(o, abc.MutableSequence):
×
332
                open_collection, close_collection = "[]"
×
333
            elif isinstance(o, abc.Set):
×
334
                open_collection, close_collection = "{}"
×
335
            else:
336
                open_collection, close_collection = "()"
×
337

338
            _write(open_collection + linesep)
×
339
            for i in o:
×
340
                _write_repr(i, indent=True, level=level)
×
341
                _write("," + linesep)
×
342
            _write(pad + close_collection)
×
343
        else:
344
            _write(repr(o))  # Numbers and bools.
×
345

346
    _write_repr(obj)
×
347
    return output.getvalue()
×
348

349

350
def rules():
3✔
351
    return (*collect_rules(), *setuptools.rules(), *pex_rules())
3✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc