• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

pantsbuild / pants / 19015773527

02 Nov 2025 05:33PM UTC coverage: 17.872% (-62.4%) from 80.3%
19015773527

Pull #22816

github

web-flow
Merge a12d75757 into 6c024e162
Pull Request #22816: Update Pants internal Python to 3.14

4 of 5 new or added lines in 3 files covered. (80.0%)

28452 existing lines in 683 files now uncovered.

9831 of 55007 relevant lines covered (17.87%)

0.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.82
/src/python/pants/core/util_rules/partitions.py
1
# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2
# Licensed under the Apache License, Version 2.0 (see LICENSE).
3

4
"""Contains the "base" code for plugin APIs which require partitioning."""
5

6
from __future__ import annotations
1✔
7

8
import itertools
1✔
9
from collections.abc import Iterable
1✔
10
from dataclasses import dataclass
1✔
11
from enum import Enum
1✔
12
from typing import Generic, Protocol, TypeVar, overload
1✔
13

14
from pants.core.goals.multi_tool_goal_helper import SkippableSubsystem
1✔
15
from pants.engine.collection import Collection
1✔
16
from pants.engine.engine_aware import EngineAwareParameter
1✔
17
from pants.engine.internals.graph import resolve_source_paths
1✔
18
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
1✔
19
from pants.engine.target import FieldSet, SourcesField, SourcesPathsRequest
1✔
20
from pants.util.memo import memoized
1✔
21
from pants.util.meta import runtime_ignore_subscripts
1✔
22

23
_FieldSetT = TypeVar("_FieldSetT", bound=FieldSet)
1✔
24

25

26
class PartitionerType(Enum):
1✔
27
    """What type of partitioner to use to partition the input specs."""
28

29
    CUSTOM = "custom"
1✔
30
    """The plugin author has a rule to go from `RequestType.PartitionRequest` -> `Partitions`."""
1✔
31

32
    DEFAULT_SINGLE_PARTITION = "default_single_partition"
1✔
33
    """Registers a partitioner which returns the inputs as a single partition
1✔
34

35
    The returned partition will have no metadata.
36
    """
37

38
    DEFAULT_ONE_PARTITION_PER_INPUT = "default_one_partition_per_input"
1✔
39
    """Registers a partitioner which returns a single-element partition per input.
1✔
40

41
    Each of the returned partitions will have no metadata.
42
    """
43

44
    def default_rules(self, cls, *, by_file: bool) -> Iterable:
1✔
45
        """Return an iterable of rules defining the default partitioning logic for this
46
        `PartitionerType`.
47

48
        NOTE: Not all `PartitionerType`s have default logic, so this method can return an empty iterable.
49

50
        :param by_file: If `True`, rules returned from this method (if any) will compute partitions with
51
          `str`-type elements, where each `str` value is the path to the resolved source field. If `False`, rule will compute
52
          partitions with `FieldSet`-type elements.
53
        """
UNCOV
54
        if self == PartitionerType.CUSTOM:
×
55
            # No default rules.
UNCOV
56
            return
×
UNCOV
57
        elif self == PartitionerType.DEFAULT_SINGLE_PARTITION:
×
UNCOV
58
            rules_generator = (
×
59
                _single_partition_file_rules if by_file else _single_partition_field_set_rules
60
            )
UNCOV
61
            yield from rules_generator(cls)
×
UNCOV
62
        elif self == PartitionerType.DEFAULT_ONE_PARTITION_PER_INPUT:
×
UNCOV
63
            rules_generator = (
×
64
                _partition_per_input_file_rules if by_file else _partition_per_input_field_set_rules
65
            )
UNCOV
66
            yield from rules_generator(cls)
×
67
        else:
68
            raise NotImplementedError(f"Partitioner type {self} is missing default rules!")
×
69

70

71
class PartitionMetadata(Protocol):
1✔
72
    @property
73
    def description(self) -> str | None: ...
74

75

76
class _EmptyMetadata:
1✔
77
    @property
1✔
78
    def description(self) -> None:
1✔
UNCOV
79
        return None
×
80

81

82
PartitionMetadataT = TypeVar("PartitionMetadataT", bound=PartitionMetadata)
1✔
83
PartitionElementT = TypeVar("PartitionElementT")
1✔
84

85

86
@dataclass(frozen=True)
1✔
87
@runtime_ignore_subscripts
1✔
88
class Partition(Generic[PartitionElementT, PartitionMetadataT]):
1✔
89
    """A collection of 'compatible' inputs for a plugin tool, with optional common metadata.
90

91
    Inputs are 'compatible' if it is safe/possible for them to be processed in the same invocation
92
    of a tool (i.e. two files formatted in the same run of a formatter, or two test targets executed
93
    in a single test runner process).
94

95
    The metadata in a partition (if any) can be any type able to cross a rule boundary, and will be
96
    provided to the rule which "runs" your tool. If it isn't `None` it must implement the
97
    `PartitionMetadata` protocol.
98

99
    NOTE: Partitions may be further divided into multiple batches before being passed to the tool-running
100
    rule. When this happens, the same metadata is passed along with each batch.
101
    """
102

103
    elements: tuple[PartitionElementT, ...]
1✔
104
    metadata: PartitionMetadataT
1✔
105

106

107
@runtime_ignore_subscripts
1✔
108
class Partitions(Collection[Partition[PartitionElementT, PartitionMetadataT]]):
1✔
109
    """A collection of (<compatible inputs>, <metadata>) pairs.
110

111
    When implementing a plugin, one of your rules will return this type, taking in a
112
    `PartitionRequest` specific to your plugin.
113

114
    The return likely will fit into one of:
115
        - Returning empty partitions: E.g. if your tool is being skipped.
116
        - Returning one partition. The partition may contain all of the inputs
117
            (as will likely be the case for target-based plugins) or a subset (which will likely be the
118
            case for targetless plugins).
119
        - Returning >1 partition. This might be the case if you can't run
120
            the tool on all the inputs at once. E.g. having to run a Python tool on XYZ with Py3,
121
            and files ABC with Py2.
122
    """
123

124
    @overload
125
    @classmethod
126
    def single_partition(
127
        cls, elements: Iterable[PartitionElementT]
128
    ) -> Partitions[PartitionElementT, _EmptyMetadata]: ...
129

130
    @overload
131
    @classmethod
132
    def single_partition(
133
        cls, elements: Iterable[PartitionElementT], *, metadata: PartitionMetadataT
134
    ) -> Partitions[PartitionElementT, PartitionMetadataT]: ...
135

136
    @classmethod
1✔
137
    def single_partition(
1✔
138
        cls, elements: Iterable[PartitionElementT], *, metadata: PartitionMetadataT | None = None
139
    ) -> Partitions:
140
        """Helper constructor for implementations that have only one partition."""
UNCOV
141
        return Partitions([Partition(tuple(elements), metadata or _EmptyMetadata())])
×
142

143

144
@dataclass(frozen=True)
1✔
145
@runtime_ignore_subscripts
1✔
146
class _BatchBase(Generic[PartitionElementT, PartitionMetadataT], EngineAwareParameter):
1✔
147
    """Base class for a collection of elements that should all be processed together.
148

149
    For example, a collection of strings pointing to files that should be linted in one process, or
150
    a collection of field-sets pointing at tests that should all execute in the same process.
151
    """
152

153
    tool_name: str
1✔
154
    elements: tuple[PartitionElementT, ...]
1✔
155
    partition_metadata: PartitionMetadataT
1✔
156

157

158
@dataclass(frozen=True)
1✔
159
@runtime_ignore_subscripts
1✔
160
class _PartitionFieldSetsRequestBase(Generic[_FieldSetT], EngineAwareParameter):
1✔
161
    """Returns a unique type per calling type.
162

163
    This serves us 2 purposes:
164
        1. `<Core Defined Plugin Type>.PartitionRequest` is the unique type used as a union base for plugin registration.
165
        2. `<Plugin Defined Subclass>.PartitionRequest` is the unique type used as the union member.
166
    """
167

168
    field_sets: tuple[_FieldSetT, ...]
1✔
169

170

171
@dataclass(frozen=True)
1✔
172
class _PartitionFilesRequestBase(EngineAwareParameter):
1✔
173
    """Returns a unique type per calling type.
174

175
    This serves us 2 purposes:
176
        1. `<Core Defined Plugin Type>.PartitionRequest` is the unique type used as a union base for plugin registration.
177
        2. `<Plugin Defined Subclass>.PartitionRequest` is the unique type used as the union member.
178
    """
179

180
    files: tuple[str, ...]
1✔
181

182

183
@memoized
1✔
184
def _single_partition_field_set_rules(cls) -> Iterable:
1✔
185
    """Returns a rule that implements a "partitioner" for `PartitionFieldSetsRequest`, which returns
186
    one partition."""
187

UNCOV
188
    @rule(
×
189
        canonical_name_suffix=cls.__name__,
190
        _param_type_overrides={
191
            "request": cls.PartitionRequest,
192
            "subsystem": cls.tool_subsystem,
193
        },
194
    )
UNCOV
195
    async def partitioner(
×
196
        request: _PartitionFieldSetsRequestBase, subsystem: SkippableSubsystem
197
    ) -> Partitions[FieldSet, _EmptyMetadata]:
198
        return Partitions() if subsystem.skip else Partitions.single_partition(request.field_sets)
×
199

UNCOV
200
    return collect_rules(locals())
×
201

202

203
@memoized
1✔
204
def _single_partition_file_rules(cls) -> Iterable:
1✔
205
    """Returns a rule that implements a "partitioner" for `PartitionFieldSetsRequest`, which returns
206
    one partition."""
207

208
    # NB: This only works if the FieldSet has a single `SourcesField` field. We check here for
209
    # a better user experience.
UNCOV
210
    sources_field_name = _get_sources_field_name(cls.field_set_type)
×
211

UNCOV
212
    @rule(
×
213
        canonical_name_suffix=cls.__name__,
214
        _param_type_overrides={
215
            "request": cls.PartitionRequest,
216
            "subsystem": cls.tool_subsystem,
217
        },
218
    )
UNCOV
219
    async def partitioner(
×
220
        request: _PartitionFieldSetsRequestBase, subsystem: SkippableSubsystem
221
    ) -> Partitions[str, _EmptyMetadata]:
222
        assert sources_field_name is not None
×
223
        all_sources_paths = await concurrently(
×
224
            resolve_source_paths(
225
                SourcesPathsRequest(getattr(field_set, sources_field_name)), **implicitly()
226
            )
227
            for field_set in request.field_sets
228
        )
229

230
        return (
×
231
            Partitions()
232
            if subsystem.skip
233
            else Partitions.single_partition(
234
                itertools.chain.from_iterable(
235
                    sources_paths.files for sources_paths in all_sources_paths
236
                )
237
            )
238
        )
239

UNCOV
240
    return collect_rules(locals())
×
241

242

243
@memoized
1✔
244
def _partition_per_input_field_set_rules(cls) -> Iterable:
1✔
245
    """Returns a rule that implements a "partitioner" for `PartitionFieldSetsRequest`, which returns
246
    a single-element partition per input."""
247

UNCOV
248
    @rule(
×
249
        canonical_name_suffix=cls.__name__,
250
        _param_type_overrides={
251
            "request": cls.PartitionRequest,
252
            "subsystem": cls.tool_subsystem,
253
        },
254
    )
UNCOV
255
    async def partitioner(
×
256
        request: _PartitionFieldSetsRequestBase, subsystem: SkippableSubsystem
257
    ) -> Partitions[FieldSet, _EmptyMetadata]:
258
        return (
×
259
            Partitions()
260
            if subsystem.skip
261
            else Partitions(
262
                Partition((field_set,), _EmptyMetadata()) for field_set in request.field_sets
263
            )
264
        )
265

UNCOV
266
    return collect_rules(locals())
×
267

268

269
@memoized
1✔
270
def _partition_per_input_file_rules(cls) -> Iterable:
1✔
271
    """Returns a rule that implements a "partitioner" for `PartitionFieldSetsRequest`, which returns
272
    a single-element partition per input."""
273

274
    # NB: This only works if the FieldSet has a single `SourcesField` field. We check here for
275
    # a better user experience.
UNCOV
276
    sources_field_name = _get_sources_field_name(cls.field_set_type)
×
277

UNCOV
278
    @rule(
×
279
        canonical_name_suffix=cls.__name__,
280
        _param_type_overrides={
281
            "request": cls.PartitionRequest,
282
            "subsystem": cls.tool_subsystem,
283
        },
284
    )
UNCOV
285
    async def partitioner(
×
286
        request: _PartitionFieldSetsRequestBase, subsystem: SkippableSubsystem
287
    ) -> Partitions[str, _EmptyMetadata]:
288
        assert sources_field_name is not None
×
289
        all_sources_paths = await concurrently(
×
290
            resolve_source_paths(
291
                SourcesPathsRequest(getattr(field_set, sources_field_name)), **implicitly()
292
            )
293
            for field_set in request.field_sets
294
        )
295

296
        return (
×
297
            Partitions()
298
            if subsystem.skip
299
            else Partitions(
300
                Partition((path,), _EmptyMetadata())
301
                for path in itertools.chain.from_iterable(
302
                    sources_paths.files for sources_paths in all_sources_paths
303
                )
304
            )
305
        )
306

UNCOV
307
    return collect_rules(locals())
×
308

309

310
def _get_sources_field_name(field_set_type: type[FieldSet]) -> str:
1✔
311
    """Get the name of the one `SourcesField` belonging to the given target type.
312

313
    NOTE: The input target type's fields must contain exactly one `SourcesField`.
314
    Otherwise this method will raise a `TypeError`.
315
    """
316

UNCOV
317
    sources_field_name = None
×
UNCOV
318
    for fieldname, fieldtype in field_set_type.fields.items():
×
UNCOV
319
        if issubclass(fieldtype, SourcesField):
×
UNCOV
320
            if sources_field_name is None:
×
UNCOV
321
                sources_field_name = fieldname
×
UNCOV
322
                break
×
323
            raise TypeError(
×
324
                f"Type {field_set_type} has multiple `SourcesField` fields."
325
                + " Pants can't provide a default partitioner."
326
            )
327
    else:
328
        raise TypeError(
×
329
            f"Type {field_set_type} has does not have a `SourcesField` field."
330
            + " Pants can't provide a default partitioner."
331
        )
UNCOV
332
    return sources_field_name
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc