25443604553

Committed 06 May 2026 03:05PM UTC coverage: 92.879% (-0.04%) from 92.915%

Build # 25443604553

Build Type

push

github

Committed by

web-flow

Commit Message

[pants_ng] Scaffolding for a pants_ng mode. (#23319)

In this mode the command line is parsed as an
NG invocation, and dispatched appropriately.

Of course at the moment there are no
implementations to dispatch to. That will follow.

This does expose a new option, `pants_ng` to users. 
There is a big warning not to set it, but we're not trying
to hide that we're working on a new thing, so I am
comfortable with this.

Coverage Stats

25 of 76 new or added lines in 9 files covered. (32.89%)

1294 existing lines in 76 files now uncovered.

92234 of 99306 relevant lines covered (92.88%)

4.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

100.0

/src/python/pants/core/goals/multi_tool_goal_helper.py

# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import logging
import os.path
from collections.abc import Iterable, Mapping, Sequence
from typing import Protocol, TypeVar

from pants.core.util_rules.distdir import DistDir
from pants.engine.fs import EMPTY_DIGEST, Digest, Workspace
from pants.option.option_types import IntOption, SkipOption, StrListOption
from pants.util.strutil import path_safe, softwrap

logger = logging.getLogger(__name__)


class SkippableSubsystem(Protocol):
    options_scope: str
    skip: SkipOption


class OnlyOption(StrListOption):
    """An --only option to select a subset of applicable tools."""

    def __new__(cls, tool_description: str, example1: str, example2: str):
        return super().__new__(
            cls,
            "--only",
            help=lambda cls: softwrap(
                f"""
                Only run these {tool_description}s and skip all others.

                The {tool_description} names are outputted at the final summary of running this goal,
                e.g. `{example1}` and `{example2}`. You can also run `{cls.name} --only=fake` to
                get a list of all activated {tool_description}s.

                You can repeat this option, e.g. `{cls.name} --only={example1} --only={example2}` or
                `{cls.name} --only=['{example1}', '{example2}']`.
                """
            ),
        )


class BatchSizeOption(IntOption):
    """A --batch-size option to help with caching tool runs."""

    def __new__(cls, uppercase: str, lowercase: str):
        return super().__new__(
            cls,
            "--batch-size",
            advanced=True,
            default=128,
            help=softwrap(
                f"""
                The target number of files to be included in each {lowercase} batch.

                {uppercase} processes are batched for a few reasons:

                  1. to avoid OS argument length limits (in processes which don't support argument files)
                  2. to support more stable cache keys than would be possible if all files were operated \
                     on in a single batch.
                  3. to allow for parallelism in {lowercase} processes which don't have internal \
                     parallelism, or -- if they do support internal parallelism -- to improve scheduling \
                     behavior when multiple processes are competing for cores and so internal \
                     parallelism cannot be used perfectly.

                In order to improve cache hit rates (see 2.), batches are created at stable boundaries,
                and so this value is only a "target" batch size (rather than an exact value).
                """
            ),
        )


def determine_specified_tool_ids(
    goal_name: str,
    only_option: Iterable[str],
    all_requests: Iterable[type],
) -> set[str]:
    all_valid_ids = {request.tool_id for request in all_requests}  # type: ignore[attr-defined]
    if not only_option:
        return all_valid_ids

    specified = set(only_option)
    unrecognized_names = specified - all_valid_ids
    if unrecognized_names:
        plural = (
            ("s", repr(sorted(unrecognized_names)))
            if len(unrecognized_names) > 1
            else ("", repr(next(iter(unrecognized_names))))
        )
        raise ValueError(
            softwrap(
                f"""
                Unrecognized name{plural[0]} with the option `--{goal_name}-only`: {plural[1]}

                All valid names: {sorted(all_valid_ids)}
                """
            )
        )
    return specified


class _ResultWithReport(Protocol):
    @property
    def report(self) -> Digest: ...

    @property
    def partition_description(self) -> str | None: ...


class _ResultsWithReports(Protocol):
    @property
    def results(self) -> Sequence[_ResultWithReport]: ...


_R = TypeVar("_R", bound=_ResultsWithReports)


def write_reports(
    results_by_tool_name: Mapping[str, Sequence[_ResultWithReport]],
    workspace: Workspace,
    dist_dir: DistDir,
    *,
    goal_name: str,
) -> None:
    disambiguated_dirs: set[str] = set()

    def write_report(digest: Digest, subdir: str) -> None:
        while subdir in disambiguated_dirs:
            # It's unlikely that two distinct partition descriptions will become the
            # same after path_safe(), but might as well be safe.
            subdir += "_"
        disambiguated_dirs.add(subdir)
        output_dir = str(dist_dir.relpath / goal_name / subdir)
        workspace.write_digest(digest, path_prefix=output_dir)
        logger.info(f"Wrote {goal_name} report files to {output_dir}.")

    for tool_name, results in results_by_tool_name.items():
        if len(results) == 1 and results[0].report != EMPTY_DIGEST:
            write_report(results[0].report, tool_name)
        else:
            for result in results:
                if result.report != EMPTY_DIGEST:
                    write_report(
                        result.report,
                        os.path.join(tool_name, path_safe(result.partition_description or "all")),
                    )

1	# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	12✔
5
6	import logging	12✔
7	import os.path	12✔
8	from collections.abc import Iterable, Mapping, Sequence	12✔
9	from typing import Protocol, TypeVar	12✔
10
11	from pants.core.util_rules.distdir import DistDir	12✔
12	from pants.engine.fs import EMPTY_DIGEST, Digest, Workspace	12✔
13	from pants.option.option_types import IntOption, SkipOption, StrListOption	12✔
14	from pants.util.strutil import path_safe, softwrap	12✔
15
16	logger = logging.getLogger(__name__)	12✔
17
18
19	class SkippableSubsystem(Protocol):	12✔
20	options_scope: str	12✔
21	skip: SkipOption	12✔
22
23
24	class OnlyOption(StrListOption):	12✔
25	"""An --only option to select a subset of applicable tools."""
26
27	def __new__(cls, tool_description: str, example1: str, example2: str):	12✔
28	return super().__new__(	12✔
29	cls,
30	"--only",
31	help=lambda cls: softwrap(
32	f"""
33	Only run these {tool_description}s and skip all others.
34
35	The {tool_description} names are outputted at the final summary of running this goal,
36	e.g. `{example1}` and `{example2}`. You can also run `{cls.name} --only=fake` to
37	get a list of all activated {tool_description}s.
38
39	You can repeat this option, e.g. `{cls.name} --only={example1} --only={example2}` or
40	`{cls.name} --only=['{example1}', '{example2}']`.
41	"""
42	),
43	)
44
45
46	class BatchSizeOption(IntOption):	12✔
47	"""A --batch-size option to help with caching tool runs."""
48
49	def __new__(cls, uppercase: str, lowercase: str):	12✔
50	return super().__new__(	12✔
51	cls,
52	"--batch-size",
53	advanced=True,
54	default=128,
55	help=softwrap(
56	f"""
57	The target number of files to be included in each {lowercase} batch.
58
59	{uppercase} processes are batched for a few reasons:
60
61	1. to avoid OS argument length limits (in processes which don't support argument files)
62	2. to support more stable cache keys than would be possible if all files were operated \
63	on in a single batch.
64	3. to allow for parallelism in {lowercase} processes which don't have internal \
65	parallelism, or -- if they do support internal parallelism -- to improve scheduling \
66	behavior when multiple processes are competing for cores and so internal \
67	parallelism cannot be used perfectly.
68
69	In order to improve cache hit rates (see 2.), batches are created at stable boundaries,
70	and so this value is only a "target" batch size (rather than an exact value).
71	"""
72	),
73	)
74
75
76	def determine_specified_tool_ids(	12✔
77	goal_name: str,
78	only_option: Iterable[str],
79	all_requests: Iterable[type],
80	) -> set[str]:
81	all_valid_ids = {request.tool_id for request in all_requests} # type: ignore[attr-defined]	4✔
82	if not only_option:	4✔
83	return all_valid_ids	3✔
84
85	specified = set(only_option)	4✔
86	unrecognized_names = specified - all_valid_ids	4✔
87	if unrecognized_names:	4✔
UNCOV 88	plural = (	1✔
89	("s", repr(sorted(unrecognized_names)))
90	if len(unrecognized_names) > 1
91	else ("", repr(next(iter(unrecognized_names))))
92	)
UNCOV 93	raise ValueError(	1✔
94	softwrap(
95	f"""
96	Unrecognized name{plural[0]} with the option `--{goal_name}-only`: {plural[1]}
97
98	All valid names: {sorted(all_valid_ids)}
99	"""
100	)
101	)
102	return specified	3✔
103
104
105	class _ResultWithReport(Protocol):	12✔
106	@property
107	def report(self) -> Digest: ...
108
109	@property
110	def partition_description(self) -> str \| None: ...
111
112
113	class _ResultsWithReports(Protocol):	12✔
114	@property
115	def results(self) -> Sequence[_ResultWithReport]: ...
116
117
118	_R = TypeVar("_R", bound=_ResultsWithReports)	12✔
119
120
121	def write_reports(	12✔
122	results_by_tool_name: Mapping[str, Sequence[_ResultWithReport]],
123	workspace: Workspace,
124	dist_dir: DistDir,
125	*,
126	goal_name: str,
127	) -> None:
128	disambiguated_dirs: set[str] = set()	3✔
129
130	def write_report(digest: Digest, subdir: str) -> None:	3✔
UNCOV 131	while subdir in disambiguated_dirs:	1✔
132	# It's unlikely that two distinct partition descriptions will become the
133	# same after path_safe(), but might as well be safe.
UNCOV 134	subdir += "_"	1✔
UNCOV 135	disambiguated_dirs.add(subdir)	1✔
UNCOV 136	output_dir = str(dist_dir.relpath / goal_name / subdir)	1✔
UNCOV 137	workspace.write_digest(digest, path_prefix=output_dir)	1✔
UNCOV 138	logger.info(f"Wrote {goal_name} report files to {output_dir}.")	1✔
139
140	for tool_name, results in results_by_tool_name.items():	3✔
141	if len(results) == 1 and results[0].report != EMPTY_DIGEST:	3✔
UNCOV 142	write_report(results[0].report, tool_name)	1✔
143	else:
144	for result in results:	3✔
145	if result.report != EMPTY_DIGEST:	3✔
UNCOV 146	write_report(	1✔
147	result.report,
148	os.path.join(tool_name, path_safe(result.partition_description or "all")),
149	)

pantsbuild / pants / 25443604553

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous