25443604553

Committed 06 May 2026 03:05PM UTC coverage: 92.879% (-0.04%) from 92.915%

Build # 25443604553

Build Type

push

github

Committed by

web-flow

Commit Message

[pants_ng] Scaffolding for a pants_ng mode. (#23319)

In this mode the command line is parsed as an
NG invocation, and dispatched appropriately.

Of course at the moment there are no
implementations to dispatch to. That will follow.

This does expose a new option, `pants_ng` to users. 
There is a big warning not to set it, but we're not trying
to hide that we're working on a new thing, so I am
comfortable with this.

Coverage Stats

25 of 76 new or added lines in 9 files covered. (32.89%)

1294 existing lines in 76 files now uncovered.

92234 of 99306 relevant lines covered (92.88%)

4.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/python/pants/bin/pants_runner.py

# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import logging
import os
import platform
import sys
import warnings
from collections.abc import Mapping
from dataclasses import dataclass

from packaging.version import Version

from pants.base.deprecated import warn_or_error
from pants.base.exception_sink import ExceptionSink
from pants.base.exiter import ExitCode
from pants.engine.env_vars import CompleteEnvironmentVars
from pants.engine.internals.native_engine import PyNgInvocation
from pants.init.logging import initialize_stdio, stdio_destination
from pants.init.util import init_workdir
from pants.option.bootstrap_options import BootstrapOptions
from pants.option.option_value_container import OptionValueContainer
from pants.option.options_bootstrapper import OptionsBootstrapper
from pants.util.docutil import doc_url
from pants.util.osutil import get_normalized_arch_name, macos_major_version
from pants.util.strutil import softwrap

logger = logging.getLogger(__name__)

# First version with working Python 3.11 support:
# https://github.com/pantsbuild/scie-pants/releases/tag/v0.12.2
# TODO: Likely still need to update scie-pants
MINIMUM_SCIE_PANTS_VERSION = Version("0.12.2")


@dataclass(frozen=True)
class PantsRunner:
    """A higher-level runner that delegates runs to either a LocalPantsRunner or
    RemotePantsRunner."""

    args: list[str]
    env: Mapping[str, str]

    # This could be a bootstrap option, but it's preferable to keep these very limited to make it
    # easier to make the daemon the default use case. Once the daemon lifecycle is stable enough we
    # should be able to avoid needing to kill it at all.
    def will_terminate_pantsd(self) -> bool:
        _DAEMON_KILLING_GOALS = frozenset(["kill-pantsd", "clean-all"])
        return not frozenset(self.args).isdisjoint(_DAEMON_KILLING_GOALS)

    def _should_run_with_pantsd(self, global_bootstrap_options: OptionValueContainer) -> bool:
        terminate_pantsd = self.will_terminate_pantsd()

        if terminate_pantsd:
            logger.debug(f"Pantsd terminating goal detected: {self.args}")

        # If we want concurrent pants runs, we can't have pantsd enabled.
        return (
            global_bootstrap_options.pantsd
            and not terminate_pantsd
            and not global_bootstrap_options.concurrent
        )

    @staticmethod
    def scrub_pythonpath() -> None:
        # Do not propagate any PYTHONPATH that happens to have been set in our environment
        # to our subprocesses.
        # Note that don't warn (but still scrub) if RUNNING_PANTS_FROM_SOURCES is set. This allows
        # scripts that run pants directly from sources, and therefore must set PYTHONPATH, to mute
        # this warning.
        pythonpath = os.environ.pop("PYTHONPATH", None)
        if pythonpath and not os.environ.pop("RUNNING_PANTS_FROM_SOURCES", None):
            logger.debug(f"Scrubbed PYTHONPATH={pythonpath} from the environment.")

    def run(self, start_time: float) -> ExitCode:
        self.scrub_pythonpath()

        # Create a transient OptionsBootstrapper with no args, to read the value of pants_ng from
        # config and env. We can't parse args until we know if we're ng or og.
        options_bootstrapper = OptionsBootstrapper.create(args=[], env=self.env, allow_pantsrc=True)
        pants_ng = options_bootstrapper.bootstrap_options.for_global_scope().pants_ng

        if pants_ng:
            logger.info("PantsRunner running as pants_ng")
            ng_invocation = PyNgInvocation.from_args(tuple(self.args[1:]))
            # Allow the existing logic to read flags in global position (note, these can be
            # prefixed with a scope so they are not necessarily just global options), so
            # that the engine can configure itself.
            global_flags = ng_invocation.global_flag_strings()
            options_bootstrapper = OptionsBootstrapper.create(
                args=[self.args[0], *global_flags], env=self.env, allow_pantsrc=True
            )
        else:
            ng_invocation = None
            options_bootstrapper = OptionsBootstrapper.create(
                args=self.args, env=self.env, allow_pantsrc=True
            )

        with warnings.catch_warnings(record=True):
            bootstrap_options = options_bootstrapper.bootstrap_options
            global_bootstrap_options = bootstrap_options.for_global_scope()

        BootstrapOptions.maybe_enable_stack_trampoline(global_bootstrap_options)

        # We enable logging here, and everything before it will be routed through regular
        # Python logging.
        stdin_fileno = sys.stdin.fileno()
        stdout_fileno = sys.stdout.fileno()
        stderr_fileno = sys.stderr.fileno()
        with (
            initialize_stdio(global_bootstrap_options),
            stdio_destination(
                stdin_fileno=stdin_fileno,
                stdout_fileno=stdout_fileno,
                stderr_fileno=stderr_fileno,
            ),
        ):
            run_via_scie = "SCIE" in os.environ
            enable_scie_warnings = "NO_SCIE_WARNING" not in os.environ
            scie_pants_version = os.environ.get("SCIE_PANTS_VERSION")

            if enable_scie_warnings:
                if not run_via_scie:
                    raise RuntimeError(
                        softwrap(
                            f"""
                            The `pants` launcher binary is now the only supported way of running Pants.
                            See {doc_url("docs/getting-started/installing-pants")} for details.
                            """
                        ),
                    )

                if run_via_scie and (
                    # either scie-pants is too old to communicate its version:
                    scie_pants_version is None
                    # or the version itself is too old:
                    or Version(scie_pants_version) < MINIMUM_SCIE_PANTS_VERSION
                ):
                    current_version_text = (
                        f"The current version of the `pants` launcher binary is {scie_pants_version}"
                        if scie_pants_version
                        else "Run `PANTS_BOOTSTRAP_VERSION=report pants` to see the current version of the `pants` launcher binary"
                    )
                    warn_or_error(
                        "2.25.0.dev0",
                        f"using a `pants` launcher binary older than {MINIMUM_SCIE_PANTS_VERSION}",
                        softwrap(
                            f"""
                            {current_version_text}, and see {doc_url("docs/getting-started/installing-pants#upgrading-pants")} for how to upgrade.
                            """
                        ),
                    )

            _validate_macos_version(global_bootstrap_options)

            # N.B. We inline imports to speed up the python thin client run, and avoids importing
            # engine types until after the runner has had a chance to set __PANTS_BIN_NAME.
            if self._should_run_with_pantsd(global_bootstrap_options):
                from pants.bin.remote_pants_runner import RemotePantsRunner

                try:
                    remote_runner = RemotePantsRunner(self.args, self.env, options_bootstrapper)
                    return remote_runner.run(start_time)
                except RemotePantsRunner.Fallback as e:
                    logger.warning(f"Client exception: {e!r}, falling back to non-daemon mode")

            from pants.bin.local_pants_runner import LocalPantsRunner

            # We only install signal handling via ExceptionSink if the run will execute in this process.
            ExceptionSink.install(
                log_location=init_workdir(global_bootstrap_options), pantsd_instance=False
            )
            runner = LocalPantsRunner.create(
                env=CompleteEnvironmentVars(self.env),
                working_dir=os.getcwd(),
                options_bootstrapper=options_bootstrapper,
                ng_invocation=ng_invocation,
            )
            return runner.run(start_time)


# for each architecture, indicate the first Pants version that doesn't support the given version of
# macOS, if it is (soon to be) unsupported:
_MACOS_VERSION_BECOMES_UNSUPPORTED_IN = {
    # macos-14 is currently oldest github hosted runner for arm
    "arm64": {
        10: "2.24.0.dev0",
        11: "2.24.0.dev0",
        12: "2.25.0.dev0",
        13: "2.25.0.dev0",
        # adding new values here should update the phrasing of the message below
    },
    # macos-13 will soon be the oldest (and only) github hosted runner for x86-64 (see https://github.com/pantsbuild/pants/issues/21333)
    "x86_64": {
        10: "2.24.0.dev0",
        11: "2.24.0.dev0",
        12: "2.25.0.dev0",
        # adding new values here should update the phrasing of the message below
    },
}


def _validate_macos_version(global_bootstrap_options: OptionValueContainer) -> None:
    """Check for running on deprecated/unsupported versions of macOS, and similar."""

    macos_version = macos_major_version()
    if macos_version is None:
        # Not macOS, no validation/deprecations required!
        return

    arch_versions = _MACOS_VERSION_BECOMES_UNSUPPORTED_IN[get_normalized_arch_name()]
    unsupported_version = arch_versions.get(macos_version)

    is_permitted_deprecated_macos_version = (
        str(macos_version) in global_bootstrap_options.allow_deprecated_macos_versions
    )

    if unsupported_version is not None and not is_permitted_deprecated_macos_version:
        warn_or_error(
            unsupported_version,
            "using Pants on older macOS",
            softwrap(
                f"""
                Recent versions of Pants only support macOS 13 and newer (on x86-64) and macOS
                14 and newer (on arm64), but this machine appears older ({platform.platform()}
                implies macOS version {macos_version}). This version also isn't permitted by your
                `[GLOBAL].allow_deprecated_macos_versions` configuration
                ({global_bootstrap_options.allow_deprecated_macos_versions}).

                Either upgrade your operating system(s), or silence this message (and thus opt-in to
                potential breakage) by adding "{macos_version}" to the
                `[GLOBAL].allow_deprecated_macos_versions` list.

                If you have questions or concerns about this, please reach out to us at
                {doc_url("community/getting-help")}.
                """
            ),
        )

1	# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	import logging	×
5	import os	×
6	import platform	×
7	import sys	×
8	import warnings	×
9	from collections.abc import Mapping	×
10	from dataclasses import dataclass	×
11
12	from packaging.version import Version	×
13
14	from pants.base.deprecated import warn_or_error	×
15	from pants.base.exception_sink import ExceptionSink	×
16	from pants.base.exiter import ExitCode	×
17	from pants.engine.env_vars import CompleteEnvironmentVars	×
NEW 18	from pants.engine.internals.native_engine import PyNgInvocation	×
19	from pants.init.logging import initialize_stdio, stdio_destination	×
20	from pants.init.util import init_workdir	×
21	from pants.option.bootstrap_options import BootstrapOptions	×
22	from pants.option.option_value_container import OptionValueContainer	×
23	from pants.option.options_bootstrapper import OptionsBootstrapper	×
24	from pants.util.docutil import doc_url	×
25	from pants.util.osutil import get_normalized_arch_name, macos_major_version	×
26	from pants.util.strutil import softwrap	×
27
28	logger = logging.getLogger(__name__)	×
29
30	# First version with working Python 3.11 support:
31	# https://github.com/pantsbuild/scie-pants/releases/tag/v0.12.2
32	# TODO: Likely still need to update scie-pants
33	MINIMUM_SCIE_PANTS_VERSION = Version("0.12.2")	×
34
35
36	@dataclass(frozen=True)	×
37	class PantsRunner:	×
38	"""A higher-level runner that delegates runs to either a LocalPantsRunner or
39	RemotePantsRunner."""
40
41	args: list[str]
42	env: Mapping[str, str]
43
44	# This could be a bootstrap option, but it's preferable to keep these very limited to make it
45	# easier to make the daemon the default use case. Once the daemon lifecycle is stable enough we
46	# should be able to avoid needing to kill it at all.
47	def will_terminate_pantsd(self) -> bool:	×
48	_DAEMON_KILLING_GOALS = frozenset(["kill-pantsd", "clean-all"])	×
49	return not frozenset(self.args).isdisjoint(_DAEMON_KILLING_GOALS)	×
50
51	def _should_run_with_pantsd(self, global_bootstrap_options: OptionValueContainer) -> bool:	×
52	terminate_pantsd = self.will_terminate_pantsd()	×
53
54	if terminate_pantsd:	×
55	logger.debug(f"Pantsd terminating goal detected: {self.args}")	×
56
57	# If we want concurrent pants runs, we can't have pantsd enabled.
58	return (	×
59	global_bootstrap_options.pantsd
60	and not terminate_pantsd
61	and not global_bootstrap_options.concurrent
62	)
63
64	@staticmethod	×
65	def scrub_pythonpath() -> None:	×
66	# Do not propagate any PYTHONPATH that happens to have been set in our environment
67	# to our subprocesses.
68	# Note that don't warn (but still scrub) if RUNNING_PANTS_FROM_SOURCES is set. This allows
69	# scripts that run pants directly from sources, and therefore must set PYTHONPATH, to mute
70	# this warning.
71	pythonpath = os.environ.pop("PYTHONPATH", None)	×
72	if pythonpath and not os.environ.pop("RUNNING_PANTS_FROM_SOURCES", None):	×
73	logger.debug(f"Scrubbed PYTHONPATH={pythonpath} from the environment.")	×
74
75	def run(self, start_time: float) -> ExitCode:	×
76	self.scrub_pythonpath()	×
77
78	# Create a transient OptionsBootstrapper with no args, to read the value of pants_ng from
79	# config and env. We can't parse args until we know if we're ng or og.
NEW 80	options_bootstrapper = OptionsBootstrapper.create(args=[], env=self.env, allow_pantsrc=True)	×
NEW 81	pants_ng = options_bootstrapper.bootstrap_options.for_global_scope().pants_ng	×
82
NEW 83	if pants_ng:	×
NEW 84	logger.info("PantsRunner running as pants_ng")	×
NEW 85	ng_invocation = PyNgInvocation.from_args(tuple(self.args[1:]))	×
86	# Allow the existing logic to read flags in global position (note, these can be
87	# prefixed with a scope so they are not necessarily just global options), so
88	# that the engine can configure itself.
NEW 89	global_flags = ng_invocation.global_flag_strings()	×
NEW 90	options_bootstrapper = OptionsBootstrapper.create(	×
91	args=[self.args[0], *global_flags], env=self.env, allow_pantsrc=True
92	)
93	else:
NEW 94	ng_invocation = None	×
NEW 95	options_bootstrapper = OptionsBootstrapper.create(	×
96	args=self.args, env=self.env, allow_pantsrc=True
97	)
98
99	with warnings.catch_warnings(record=True):	×
100	bootstrap_options = options_bootstrapper.bootstrap_options	×
101	global_bootstrap_options = bootstrap_options.for_global_scope()	×
102
103	BootstrapOptions.maybe_enable_stack_trampoline(global_bootstrap_options)	×
104
105	# We enable logging here, and everything before it will be routed through regular
106	# Python logging.
107	stdin_fileno = sys.stdin.fileno()	×
108	stdout_fileno = sys.stdout.fileno()	×
109	stderr_fileno = sys.stderr.fileno()	×
110	with (	×
111	initialize_stdio(global_bootstrap_options),
112	stdio_destination(
113	stdin_fileno=stdin_fileno,
114	stdout_fileno=stdout_fileno,
115	stderr_fileno=stderr_fileno,
116	),
117	):
118	run_via_scie = "SCIE" in os.environ	×
119	enable_scie_warnings = "NO_SCIE_WARNING" not in os.environ	×
120	scie_pants_version = os.environ.get("SCIE_PANTS_VERSION")	×
121
122	if enable_scie_warnings:	×
123	if not run_via_scie:	×
124	raise RuntimeError(	×
125	softwrap(
126	f"""
127	The `pants` launcher binary is now the only supported way of running Pants.
128	See {doc_url("docs/getting-started/installing-pants")} for details.
129	"""
130	),
131	)
132
133	if run_via_scie and (	×
134	# either scie-pants is too old to communicate its version:
135	scie_pants_version is None
136	# or the version itself is too old:
137	or Version(scie_pants_version) < MINIMUM_SCIE_PANTS_VERSION
138	):
139	current_version_text = (	×
140	f"The current version of the `pants` launcher binary is {scie_pants_version}"
141	if scie_pants_version
142	else "Run `PANTS_BOOTSTRAP_VERSION=report pants` to see the current version of the `pants` launcher binary"
143	)
144	warn_or_error(	×
145	"2.25.0.dev0",
146	f"using a `pants` launcher binary older than {MINIMUM_SCIE_PANTS_VERSION}",
147	softwrap(
148	f"""
149	{current_version_text}, and see {doc_url("docs/getting-started/installing-pants#upgrading-pants")} for how to upgrade.
150	"""
151	),
152	)
153
154	_validate_macos_version(global_bootstrap_options)	×
155
156	# N.B. We inline imports to speed up the python thin client run, and avoids importing
157	# engine types until after the runner has had a chance to set __PANTS_BIN_NAME.
158	if self._should_run_with_pantsd(global_bootstrap_options):	×
159	from pants.bin.remote_pants_runner import RemotePantsRunner	×
160
161	try:	×
162	remote_runner = RemotePantsRunner(self.args, self.env, options_bootstrapper)	×
163	return remote_runner.run(start_time)	×
164	except RemotePantsRunner.Fallback as e:	×
165	logger.warning(f"Client exception: {e!r}, falling back to non-daemon mode")	×
166
167	from pants.bin.local_pants_runner import LocalPantsRunner	×
168
169	# We only install signal handling via ExceptionSink if the run will execute in this process.
170	ExceptionSink.install(	×
171	log_location=init_workdir(global_bootstrap_options), pantsd_instance=False
172	)
173	runner = LocalPantsRunner.create(	×
174	env=CompleteEnvironmentVars(self.env),
175	working_dir=os.getcwd(),
176	options_bootstrapper=options_bootstrapper,
177	ng_invocation=ng_invocation,
178	)
179	return runner.run(start_time)	×
180
181
182	# for each architecture, indicate the first Pants version that doesn't support the given version of
183	# macOS, if it is (soon to be) unsupported:
184	_MACOS_VERSION_BECOMES_UNSUPPORTED_IN = {	×
185	# macos-14 is currently oldest github hosted runner for arm
186	"arm64": {
187	10: "2.24.0.dev0",
188	11: "2.24.0.dev0",
189	12: "2.25.0.dev0",
190	13: "2.25.0.dev0",
191	# adding new values here should update the phrasing of the message below
192	},
193	# macos-13 will soon be the oldest (and only) github hosted runner for x86-64 (see https://github.com/pantsbuild/pants/issues/21333)
194	"x86_64": {
195	10: "2.24.0.dev0",
196	11: "2.24.0.dev0",
197	12: "2.25.0.dev0",
198	# adding new values here should update the phrasing of the message below
199	},
200	}
201
202
203	def _validate_macos_version(global_bootstrap_options: OptionValueContainer) -> None:	×
204	"""Check for running on deprecated/unsupported versions of macOS, and similar."""
205
206	macos_version = macos_major_version()	×
207	if macos_version is None:	×
208	# Not macOS, no validation/deprecations required!
209	return	×
210
211	arch_versions = _MACOS_VERSION_BECOMES_UNSUPPORTED_IN[get_normalized_arch_name()]	×
212	unsupported_version = arch_versions.get(macos_version)	×
213
214	is_permitted_deprecated_macos_version = (	×
215	str(macos_version) in global_bootstrap_options.allow_deprecated_macos_versions
216	)
217
218	if unsupported_version is not None and not is_permitted_deprecated_macos_version:	×
219	warn_or_error(	×
220	unsupported_version,
221	"using Pants on older macOS",
222	softwrap(
223	f"""
224	Recent versions of Pants only support macOS 13 and newer (on x86-64) and macOS
225	14 and newer (on arm64), but this machine appears older ({platform.platform()}
226	implies macOS version {macos_version}). This version also isn't permitted by your
227	`[GLOBAL].allow_deprecated_macos_versions` configuration
228	({global_bootstrap_options.allow_deprecated_macos_versions}).
229
230	Either upgrade your operating system(s), or silence this message (and thus opt-in to
231	potential breakage) by adding "{macos_version}" to the
232	`[GLOBAL].allow_deprecated_macos_versions` list.
233
234	If you have questions or concerns about this, please reach out to us at
235	{doc_url("community/getting-help")}.
236	"""
237	),
238	)

pantsbuild / pants / 25443604553

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous