20438429929

Committed 22 Dec 2025 04:55PM UTC coverage: 80.287% (+0.003%) from 80.284%

Build # 20438429929

Build Type

Pull #22934

github

Committed by

web-flow

Commit Message

Merge b49c09e21 into 06f105be8

Pull Request Pull Request #22934: feat(go): add multi-module support to golangci-lint plugin and upgrade to v2

Run Details

37 of 62 new or added lines in 3 files covered. (59.68%)

183 existing lines in 9 files now uncovered.

78528 of 97809 relevant lines covered (80.29%)

3.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

44.0

/src/python/pants/pantsd/pants_daemon.py

# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import logging
import os
import sys
import time
import warnings
from pathlib import PurePath

from setproctitle import setproctitle as set_process_title

from pants.base.build_environment import get_buildroot
from pants.base.exception_sink import ExceptionSink
from pants.bin.daemon_pants_runner import DaemonPantsRunner
from pants.engine.internals import native_engine
from pants.engine.internals.native_engine import PyExecutor, PyNailgunServer
from pants.init.engine_initializer import GraphScheduler
from pants.init.logging import initialize_stdio, pants_log_path
from pants.init.util import init_workdir
from pants.option.bootstrap_options import LocalStoreOptions
from pants.option.global_options import GlobalOptions
from pants.option.option_value_container import OptionValueContainer
from pants.option.options import Options
from pants.option.options_bootstrapper import OptionsBootstrapper
from pants.pantsd.pants_daemon_core import PantsDaemonCore
from pants.pantsd.process_manager import PantsDaemonProcessManager
from pants.pantsd.service.pants_service import PantsServices
from pants.pantsd.service.scheduler_service import SchedulerService
from pants.pantsd.service.store_gc_service import StoreGCService
from pants.util.contextutil import argv_as, hermetic_environment_as
from pants.util.dirutil import safe_open
from pants.version import VERSION

_SHUTDOWN_TIMEOUT_SECS = 3

_PRESERVED_ENV_VARS = [
    # Provides location of sandboxer binary.
    "PANTS_SANDBOXER_BINARY_PATH",
    # Controls backtrace behavior for rust code.
    "RUST_BACKTRACE",
    # The environment variables consumed by the `bollard` crate as of
    # https://github.com/fussybeaver/bollard/commit/a12c6b21b737e5ea9e6efe5f0128d02dc594f9aa
    "DOCKER_HOST",
    "DOCKER_CONFIG",
    "DOCKER_CERT_PATH",
    # Environment variables consumed (indirectly) by the `docker_credential` crate as of
    # https://github.com/keirlawson/docker_credential/commit/0c42d0f3c76a7d5f699d4d1e8b9747f799cf6116
    "HOME",
    "PATH",
    "USER",
    # Environment variables needed to support downloading tools in a restricted internet environment
    "HTTP_PROXY",
    "HTTPS_PROXY",
    "NO_PROXY",
    "http_proxy",  # for historical reasons, in some environments, *_proxy variables must be lowercase
    "https_proxy",
    "no_proxy",
]


class PantsDaemon(PantsDaemonProcessManager):
    """A daemon that manages PantsService instances."""

    JOIN_TIMEOUT_SECONDS = 1

    class StartupFailure(Exception):
        """Represents a failure to start pantsd."""

    class RuntimeFailure(Exception):
        """Represents a pantsd failure at runtime, usually from an underlying service failure."""

    @classmethod
    def create(cls, options_bootstrapper: OptionsBootstrapper) -> PantsDaemon:
        # Any warnings that would be triggered here are re-triggered later per-run of Pants, so we
        # silence them.
        with warnings.catch_warnings(record=True):
            bootstrap_options = options_bootstrapper.bootstrap_options
            bootstrap_options_values = bootstrap_options.for_global_scope()

        # This executor is owned by the PantsDaemon, and borrowed by the Pants runs that are launched by
        # PantsDaemonCore. Individual runs will call shutdown to tear down the executor, but those calls
        # have no effect on a borrowed executor.
        executor = GlobalOptions.create_py_executor(bootstrap_options_values)
        core = PantsDaemonCore(options_bootstrapper, executor.to_borrowed(), cls._setup_services)

        server = native_engine.nailgun_server_create(
            executor,
            bootstrap_options_values.pantsd_pailgun_port,
            DaemonPantsRunner(core),
        )

        return PantsDaemon(
            work_dir=bootstrap_options_values.pants_workdir,
            executor=executor,
            server=server,
            core=core,
            bootstrap_options=bootstrap_options,
        )

    @staticmethod
    def _setup_services(
        bootstrap_options: OptionValueContainer,
        graph_scheduler: GraphScheduler,
    ):
        """Initialize pantsd services.

        :returns: A PantsServices instance.
        """
        build_root = get_buildroot()

        invalidation_globs = GlobalOptions.compute_pantsd_invalidation_globs(
            build_root,
            bootstrap_options,
        )

        scheduler_service = SchedulerService(
            graph_scheduler=graph_scheduler,
            build_root=build_root,
            invalidation_globs=invalidation_globs,
            pidfile=PantsDaemon.metadata_file_path(
                "pantsd", "pid", bootstrap_options.pants_subprocessdir
            ),
            pid=os.getpid(),
            max_memory_usage_in_bytes=bootstrap_options.pantsd_max_memory_usage,
        )

        store_gc_service = StoreGCService(
            graph_scheduler.scheduler,
            local_store_options=LocalStoreOptions.from_options(bootstrap_options),
        )
        return PantsServices(services=(scheduler_service, store_gc_service))

    def __init__(
        self,
        work_dir: str,
        executor: PyExecutor,
        server: PyNailgunServer,
        core: PantsDaemonCore,
        bootstrap_options: Options,
    ):
        """
        NB: A PantsDaemon instance is generally instantiated via `create`.
        """
        super().__init__(bootstrap_options, daemon_entrypoint=__name__)
        self._build_root = get_buildroot()
        self._work_dir = work_dir
        self._executor = executor
        self._server = server
        self._core = core
        self._bootstrap_options = bootstrap_options

        self._logger = logging.getLogger(__name__)

    def _close_stdio(self, log_path: PurePath):
        """Close stdio and append to a log path instead.

        The vast majority of Python-level IO will be re-routed to thread-local destinations by
        `initialize_stdio`, but we close stdio to avoid any stray output in the tty that launched
        pantsd.

        Rather than leaving 0, 1, 2 dangling though, we open replacements as a backstop for fatal
        errors or unmodified code (such as Rust panic handlers) that might expect them to be valid
        file handles.
        """
        for attr, writable in (("stdin", False), ("stdout", True), ("stderr", True)):
            # Close the old.
            fd = getattr(sys, attr)
            fileno = fd.fileno()
            fd.flush()
            fd.close()

            # Open the new.
            temp_fd = safe_open(log_path, "a") if writable else open(os.devnull)
            os.dup2(temp_fd.fileno(), fileno)
            setattr(sys, attr, os.fdopen(fileno, mode=("w" if writable else "r")))
        sys.__stdin__, sys.__stdout__, sys.__stderr__ = sys.stdin, sys.stdout, sys.stderr  # type: ignore[assignment,misc]

    def _initialize_metadata(self, options_fingerprint: str) -> None:
        """Writes out our pid and other metadata.

        Order matters a bit here, because technically all that is necessary to connect is the port,
        and Services are lazily initialized by the core when a connection is established. Our pid
        needs to be on disk before that happens.
        """

        # Write the pidfile. The SchedulerService will monitor it after a grace period.
        self.write_pid()
        self.write_process_name()
        self.write_fingerprint(options_fingerprint)
        self._logger.info(f"pantsd {VERSION} running with PID: {self.pid}")
        self.write_socket(self._server.port())

    def run_sync(self):
        """Synchronously run pantsd."""
        os.environ.pop("PYTHONPATH")

        global_bootstrap_options = self._bootstrap_options.for_global_scope()
        options_fingerprint = self.options_fingerprint
        # Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
        set_process_title(f"pantsd [{self._build_root}]")

        # Switch log output to the daemon's log stream, and empty `env` and `argv` to encourage all
        # further usage of those variables to happen via engine APIs and options.
        self._close_stdio(pants_log_path(PurePath(global_bootstrap_options.pants_workdir)))
        with (
            initialize_stdio(global_bootstrap_options),
            argv_as(tuple()),
            hermetic_environment_as(*_PRESERVED_ENV_VARS),
        ):
            # Install signal and panic handling.
            ExceptionSink.install(
                log_location=init_workdir(global_bootstrap_options), pantsd_instance=True
            )
            native_engine.maybe_set_panic_handler()

            self._initialize_metadata(options_fingerprint)

            # Check periodically whether the core is valid, and exit if it is not.
            while self._core.is_valid():
                time.sleep(self.JOIN_TIMEOUT_SECONDS)

            # We're exiting: purge our metadata to prevent new connections, then join the server
            # to avoid interrupting ongoing runs.
            self.purge_metadata(force=True)
            self._logger.info("Waiting for ongoing runs to complete before exiting...")
            native_engine.nailgun_server_await_shutdown(self._server)

            # Shutdown the PantsDaemonCore, which will shut down any live Scheduler.
            self._logger.info("Waiting for Sessions to complete before exiting...")
            self._core.shutdown()

            # Shutdown the executor. The shutdown method will log if that takes an unexpected
            # amount of time, so we only log at debug here.
            self._logger.debug("Waiting for tasks to complete before exiting...")
            self._executor.shutdown(_SHUTDOWN_TIMEOUT_SECS)

            self._logger.info("Exiting pantsd")


def launch_new_pantsd_instance():
    """An external entrypoint that spawns a new pantsd instance."""

    options_bootstrapper = OptionsBootstrapper.create(
        args=sys.argv, env=os.environ, allow_pantsrc=True
    )
    daemon = PantsDaemon.create(options_bootstrapper)
    daemon.run_sync()

1	# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	12✔
5
6	import logging	12✔
7	import os	12✔
8	import sys	12✔
9	import time	12✔
10	import warnings	12✔
11	from pathlib import PurePath	12✔
12
13	from setproctitle import setproctitle as set_process_title	12✔
14
15	from pants.base.build_environment import get_buildroot	12✔
16	from pants.base.exception_sink import ExceptionSink	12✔
17	from pants.bin.daemon_pants_runner import DaemonPantsRunner	12✔
18	from pants.engine.internals import native_engine	12✔
19	from pants.engine.internals.native_engine import PyExecutor, PyNailgunServer	12✔
20	from pants.init.engine_initializer import GraphScheduler	12✔
21	from pants.init.logging import initialize_stdio, pants_log_path	12✔
22	from pants.init.util import init_workdir	12✔
23	from pants.option.bootstrap_options import LocalStoreOptions	12✔
24	from pants.option.global_options import GlobalOptions	12✔
25	from pants.option.option_value_container import OptionValueContainer	12✔
26	from pants.option.options import Options	12✔
27	from pants.option.options_bootstrapper import OptionsBootstrapper	12✔
28	from pants.pantsd.pants_daemon_core import PantsDaemonCore	12✔
29	from pants.pantsd.process_manager import PantsDaemonProcessManager	12✔
30	from pants.pantsd.service.pants_service import PantsServices	12✔
31	from pants.pantsd.service.scheduler_service import SchedulerService	12✔
32	from pants.pantsd.service.store_gc_service import StoreGCService	12✔
33	from pants.util.contextutil import argv_as, hermetic_environment_as	12✔
34	from pants.util.dirutil import safe_open	12✔
35	from pants.version import VERSION	12✔
36
37	_SHUTDOWN_TIMEOUT_SECS = 3	12✔
38
39	_PRESERVED_ENV_VARS = [	12✔
40	# Provides location of sandboxer binary.
41	"PANTS_SANDBOXER_BINARY_PATH",
42	# Controls backtrace behavior for rust code.
43	"RUST_BACKTRACE",
44	# The environment variables consumed by the `bollard` crate as of
45	# https://github.com/fussybeaver/bollard/commit/a12c6b21b737e5ea9e6efe5f0128d02dc594f9aa
46	"DOCKER_HOST",
47	"DOCKER_CONFIG",
48	"DOCKER_CERT_PATH",
49	# Environment variables consumed (indirectly) by the `docker_credential` crate as of
50	# https://github.com/keirlawson/docker_credential/commit/0c42d0f3c76a7d5f699d4d1e8b9747f799cf6116
51	"HOME",
52	"PATH",
53	"USER",
54	# Environment variables needed to support downloading tools in a restricted internet environment
55	"HTTP_PROXY",
56	"HTTPS_PROXY",
57	"NO_PROXY",
58	"http_proxy", # for historical reasons, in some environments, *_proxy variables must be lowercase
59	"https_proxy",
60	"no_proxy",
61	]
62
63
64	class PantsDaemon(PantsDaemonProcessManager):	12✔
65	"""A daemon that manages PantsService instances."""
66
67	JOIN_TIMEOUT_SECONDS = 1	12✔
68
69	class StartupFailure(Exception):	12✔
70	"""Represents a failure to start pantsd."""
71
72	class RuntimeFailure(Exception):	12✔
73	"""Represents a pantsd failure at runtime, usually from an underlying service failure."""
74
75	@classmethod	12✔
76	def create(cls, options_bootstrapper: OptionsBootstrapper) -> PantsDaemon:	12✔
77	# Any warnings that would be triggered here are re-triggered later per-run of Pants, so we
78	# silence them.
79	with warnings.catch_warnings(record=True):	×
80	bootstrap_options = options_bootstrapper.bootstrap_options	×
UNCOV 81	bootstrap_options_values = bootstrap_options.for_global_scope()	×
82
83	# This executor is owned by the PantsDaemon, and borrowed by the Pants runs that are launched by
84	# PantsDaemonCore. Individual runs will call shutdown to tear down the executor, but those calls
85	# have no effect on a borrowed executor.
UNCOV 86	executor = GlobalOptions.create_py_executor(bootstrap_options_values)	×
UNCOV 87	core = PantsDaemonCore(options_bootstrapper, executor.to_borrowed(), cls._setup_services)	×
88
UNCOV 89	server = native_engine.nailgun_server_create(	×
90	executor,
91	bootstrap_options_values.pantsd_pailgun_port,
92	DaemonPantsRunner(core),
93	)
94
UNCOV 95	return PantsDaemon(	×
96	work_dir=bootstrap_options_values.pants_workdir,
97	executor=executor,
98	server=server,
99	core=core,
100	bootstrap_options=bootstrap_options,
101	)
102
103	@staticmethod	12✔
104	def _setup_services(	12✔
105	bootstrap_options: OptionValueContainer,
106	graph_scheduler: GraphScheduler,
107	):
108	"""Initialize pantsd services.
109
110	:returns: A PantsServices instance.
111	"""
112	build_root = get_buildroot()	×
113
UNCOV 114	invalidation_globs = GlobalOptions.compute_pantsd_invalidation_globs(	×
115	build_root,
116	bootstrap_options,
117	)
118
UNCOV 119	scheduler_service = SchedulerService(	×
120	graph_scheduler=graph_scheduler,
121	build_root=build_root,
122	invalidation_globs=invalidation_globs,
123	pidfile=PantsDaemon.metadata_file_path(
124	"pantsd", "pid", bootstrap_options.pants_subprocessdir
125	),
126	pid=os.getpid(),
127	max_memory_usage_in_bytes=bootstrap_options.pantsd_max_memory_usage,
128	)
129
UNCOV 130	store_gc_service = StoreGCService(	×
131	graph_scheduler.scheduler,
132	local_store_options=LocalStoreOptions.from_options(bootstrap_options),
133	)
UNCOV 134	return PantsServices(services=(scheduler_service, store_gc_service))	×
135
136	def __init__(	12✔
137	self,
138	work_dir: str,
139	executor: PyExecutor,
140	server: PyNailgunServer,
141	core: PantsDaemonCore,
142	bootstrap_options: Options,
143	):
144	"""
145	NB: A PantsDaemon instance is generally instantiated via `create`.
146	"""
UNCOV 147	super().__init__(bootstrap_options, daemon_entrypoint=__name__)	×
148	self._build_root = get_buildroot()	×
UNCOV 149	self._work_dir = work_dir	×
UNCOV 150	self._executor = executor	×
UNCOV 151	self._server = server	×
UNCOV 152	self._core = core	×
UNCOV 153	self._bootstrap_options = bootstrap_options	×
154
UNCOV 155	self._logger = logging.getLogger(__name__)	×
156
157	def _close_stdio(self, log_path: PurePath):	12✔
158	"""Close stdio and append to a log path instead.
159
160	The vast majority of Python-level IO will be re-routed to thread-local destinations by
161	`initialize_stdio`, but we close stdio to avoid any stray output in the tty that launched
162	pantsd.
163
164	Rather than leaving 0, 1, 2 dangling though, we open replacements as a backstop for fatal
165	errors or unmodified code (such as Rust panic handlers) that might expect them to be valid
166	file handles.
167	"""
UNCOV 168	for attr, writable in (("stdin", False), ("stdout", True), ("stderr", True)):	×
169	# Close the old.
170	fd = getattr(sys, attr)	×
171	fileno = fd.fileno()	×
172	fd.flush()	×
UNCOV 173	fd.close()	×
174
175	# Open the new.
UNCOV 176	temp_fd = safe_open(log_path, "a") if writable else open(os.devnull)	×
UNCOV 177	os.dup2(temp_fd.fileno(), fileno)	×
UNCOV 178	setattr(sys, attr, os.fdopen(fileno, mode=("w" if writable else "r")))	×
UNCOV 179	sys.__stdin__, sys.__stdout__, sys.__stderr__ = sys.stdin, sys.stdout, sys.stderr # type: ignore[assignment,misc]	×
180
181	def _initialize_metadata(self, options_fingerprint: str) -> None:	12✔
182	"""Writes out our pid and other metadata.
183
184	Order matters a bit here, because technically all that is necessary to connect is the port,
185	and Services are lazily initialized by the core when a connection is established. Our pid
186	needs to be on disk before that happens.
187	"""
188
189	# Write the pidfile. The SchedulerService will monitor it after a grace period.
UNCOV 190	self.write_pid()	×
191	self.write_process_name()	×
UNCOV 192	self.write_fingerprint(options_fingerprint)	×
193	self._logger.info(f"pantsd {VERSION} running with PID: {self.pid}")	×
194	self.write_socket(self._server.port())	×
195
196	def run_sync(self):	12✔
197	"""Synchronously run pantsd."""
UNCOV 198	os.environ.pop("PYTHONPATH")	×
199
200	global_bootstrap_options = self._bootstrap_options.for_global_scope()	×
201	options_fingerprint = self.options_fingerprint	×
202	# Set the process name in ps output to 'pantsd' vs './pants compile src/etc:: -ldebug'.
UNCOV 203	set_process_title(f"pantsd [{self._build_root}]")	×
204
205	# Switch log output to the daemon's log stream, and empty `env` and `argv` to encourage all
206	# further usage of those variables to happen via engine APIs and options.
207	self._close_stdio(pants_log_path(PurePath(global_bootstrap_options.pants_workdir)))	×
UNCOV 208	with (	×
209	initialize_stdio(global_bootstrap_options),
210	argv_as(tuple()),
211	hermetic_environment_as(*_PRESERVED_ENV_VARS),
212	):
213	# Install signal and panic handling.
UNCOV 214	ExceptionSink.install(	×
215	log_location=init_workdir(global_bootstrap_options), pantsd_instance=True
216	)
UNCOV 217	native_engine.maybe_set_panic_handler()	×
218
UNCOV 219	self._initialize_metadata(options_fingerprint)	×
220
221	# Check periodically whether the core is valid, and exit if it is not.
222	while self._core.is_valid():	×
UNCOV 223	time.sleep(self.JOIN_TIMEOUT_SECONDS)	×
224
225	# We're exiting: purge our metadata to prevent new connections, then join the server
226	# to avoid interrupting ongoing runs.
UNCOV 227	self.purge_metadata(force=True)	×
UNCOV 228	self._logger.info("Waiting for ongoing runs to complete before exiting...")	×
UNCOV 229	native_engine.nailgun_server_await_shutdown(self._server)	×
230
231	# Shutdown the PantsDaemonCore, which will shut down any live Scheduler.
UNCOV 232	self._logger.info("Waiting for Sessions to complete before exiting...")	×
233	self._core.shutdown()	×
234
235	# Shutdown the executor. The shutdown method will log if that takes an unexpected
236	# amount of time, so we only log at debug here.
UNCOV 237	self._logger.debug("Waiting for tasks to complete before exiting...")	×
UNCOV 238	self._executor.shutdown(_SHUTDOWN_TIMEOUT_SECS)	×
239
UNCOV 240	self._logger.info("Exiting pantsd")	×
241
242
243	def launch_new_pantsd_instance():	12✔
244	"""An external entrypoint that spawns a new pantsd instance."""
245
UNCOV 246	options_bootstrapper = OptionsBootstrapper.create(	×
247	args=sys.argv, env=os.environ, allow_pantsrc=True
248	)
UNCOV 249	daemon = PantsDaemon.create(options_bootstrapper)	×
UNCOV 250	daemon.run_sync()	×

pantsbuild / pants / 20438429929

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous