20328535594

Committed 18 Dec 2025 06:46AM UTC coverage: 57.969% (-22.3%) from 80.295%

Build # 20328535594

Build Type

Pull #22954

github

Committed by

web-flow

Commit Message

Merge ccc9c5409 into 407284c67

Pull Request Pull Request #22954: free up disk space in runner image

Run Details

39083 of 67421 relevant lines covered (57.97%)

0.91 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

38.07

/src/python/pants/base/exception_sink.py

# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import atexit
import datetime
import faulthandler
import logging
import os
import signal
import sys
import threading
import traceback
from collections.abc import Callable, Iterator
from contextlib import contextmanager

import psutil
import setproctitle

from pants.util.dirutil import safe_mkdir, safe_open
from pants.util.osutil import Pid

logger = logging.getLogger(__name__)


class SignalHandler:
    """A specification for how to handle a fixed set of nonfatal signals.

    This is subclassed and registered with ExceptionSink.reset_signal_handler() whenever the signal
    handling behavior is modified for different pants processes, for example in the remote client when
    pantsd is enabled. The default behavior is to exit "gracefully" by leaving a detailed log of which
    signal was received, then exiting with failure.

    Note that the terminal converts a ctrl-c from the user into a SIGINT.
    """

    @property
    def signal_handler_mapping(self) -> dict[signal.Signals, Callable]:
        """A dict mapping (signal number) -> (a method handling the signal)."""
        # Could use an enum here, but we never end up doing any matching on the specific signal value,
        # instead just iterating over the registered signals to set handlers, so a dict is probably
        # better.
        return {
            signal.SIGINT: self._handle_sigint_if_enabled,
            signal.SIGQUIT: self.handle_sigquit,
            signal.SIGTERM: self.handle_sigterm,
        }

    def __init__(self, *, pantsd_instance: bool):
        self._ignore_sigint_lock = threading.Lock()
        self._ignoring_sigint = False
        self._pantsd_instance = pantsd_instance

    def _handle_sigint_if_enabled(self, signum: int, _frame):
        with self._ignore_sigint_lock:
            if not self._ignoring_sigint:
                self.handle_sigint(signum, _frame)

    def _toggle_ignoring_sigint(self, toggle: bool) -> None:
        if not self._pantsd_instance:
            with self._ignore_sigint_lock:
                self._ignoring_sigint = toggle

    def _send_signal_to_children(self, received_signal: int, signame: str) -> None:
        """Send a signal to any children of this process in order.

        Pants may have spawned multiple subprocesses via Python or Rust. Upon receiving a signal,
        this method is invoked to propagate the signal to all children, regardless of how they were
        spawned.
        """

        self_process = psutil.Process()
        children = self_process.children()
        logger.debug(f"Sending signal {signame} ({received_signal}) to child processes: {children}")
        for child_process in children:
            child_process.send_signal(received_signal)

    def handle_sigint(self, signum: int, _frame):
        self._send_signal_to_children(signum, "SIGINT")
        raise KeyboardInterrupt("User interrupted execution with control-c!")

    # TODO(#7406): figure out how to let sys.exit work in a signal handler instead of having to raise
    # this exception!
    class SignalHandledNonLocalExit(Exception):
        """Raised in handlers for non-fatal signals to overcome Python limitations.

        When waiting on a subprocess and in a signal handler, sys.exit appears to be ignored, and
        causes the signal handler to return. We want to (eventually) exit after these signals, not
        ignore them, so we raise this exception instead and check it in our sys.excepthook override.
        """

        def __init__(self, signum, signame):
            self.signum = signum
            self.signame = signame
            self.traceback_lines = traceback.format_stack()
            super().__init__()

            if "I/O operation on closed file" in self.traceback_lines:
                logger.debug(
                    "SignalHandledNonLocalExit: unexpected appearance of "
                    "'I/O operation on closed file' in traceback"
                )

    def handle_sigquit(self, signum, _frame):
        self._send_signal_to_children(signum, "SIGQUIT")
        raise self.SignalHandledNonLocalExit(signum, "SIGQUIT")

    def handle_sigterm(self, signum, _frame):
        self._send_signal_to_children(signum, "SIGTERM")
        raise self.SignalHandledNonLocalExit(signum, "SIGTERM")


class ExceptionSink:
    """A mutable singleton object representing where exceptions should be logged to.

    The ExceptionSink should be installed in any process that is running Pants @rules via the
    engine. Notably, this does _not_ include the pantsd client, which does its own signal handling
    directly in order to forward information to the pantsd server.
    """

    # NB: see the bottom of this file where we call reset_log_location() and other mutators in order
    # to properly setup global state.
    _log_dir = None

    # Where to log stacktraces to in a SIGUSR2 handler.
    _interactive_output_stream = None

    # An instance of `SignalHandler` which is invoked to handle a static set of specific nonfatal
    # signals (these signal handlers are allowed to make pants exit, but unlike SIGSEGV they don't
    # need to exit immediately).
    _signal_handler: SignalHandler = SignalHandler(pantsd_instance=False)

    # These persistent open file descriptors are kept so the signal handler can do almost no work
    # (and lets faulthandler figure out signal safety).
    _pid_specific_error_fileobj = None
    _shared_error_fileobj = None

    def __new__(cls, *args, **kwargs):
        raise TypeError(
            f"Instances of {cls.__name__} are not allowed to be constructed! Call install() instead."
        )

    class ExceptionSinkError(Exception):
        pass

    @classmethod
    def install(cls, log_location: str, pantsd_instance: bool) -> None:
        """Setup global state for this process, such as signal handlers and sys.excepthook."""

        # Set the log location for writing logs before bootstrap options are parsed.
        cls.reset_log_location(log_location)

        # NB: Mutate process-global state!
        sys.excepthook = ExceptionSink.log_exception

        # Setup a default signal handler.
        cls.reset_signal_handler(SignalHandler(pantsd_instance=pantsd_instance))

    # All reset_* methods are ~idempotent!
    @classmethod
    def reset_log_location(cls, new_log_location: str) -> None:
        """Re-acquire file handles to error logs based in the new location.

        Class state:
        - Overwrites `cls._log_dir`, `cls._pid_specific_error_fileobj`, and
          `cls._shared_error_fileobj`.
        OS state:
        - May create a new directory.
        - Overwrites signal handlers for many fatal and non-fatal signals (but not SIGUSR2).

        :raises: :class:`ExceptionSink.ExceptionSinkError` if the directory does not exist or is not
                 writable.
        """
        # We could no-op here if the log locations are the same, but there's no reason not to have the
        # additional safety of re-acquiring file descriptors each time (and erroring out early if the
        # location is no longer writable).
        try:
            safe_mkdir(new_log_location)
        except Exception as e:
            raise cls.ExceptionSinkError(
                f"The provided log location path at '{new_log_location}' is not writable or could not be created: {str(e)}.",
                e,
            )

        pid = os.getpid()
        pid_specific_log_path = cls.exceptions_log_path(for_pid=pid, in_dir=new_log_location)
        shared_log_path = cls.exceptions_log_path(in_dir=new_log_location)
        assert pid_specific_log_path != shared_log_path
        try:
            pid_specific_error_stream = cls.open_pid_specific_error_stream(pid_specific_log_path)
            shared_error_stream = safe_open(shared_log_path, mode="a")
        except Exception as e:
            raise cls.ExceptionSinkError(
                f"Error opening fatal error log streams for log location '{new_log_location}': {str(e)}"
            )

        # NB: mutate process-global state!
        if faulthandler.is_enabled():
            logger.debug("re-enabling faulthandler")
            # Call Py_CLEAR() on the previous error stream:
            # https://github.com/vstinner/faulthandler/blob/master/faulthandler.c
            faulthandler.disable()
        # Send a stacktrace to this file if interrupted by a fatal error.
        faulthandler.enable(file=pid_specific_error_stream, all_threads=True)

        # NB: mutate the class variables!
        cls._log_dir = new_log_location
        cls._pid_specific_error_fileobj = pid_specific_error_stream
        cls._shared_error_fileobj = shared_error_stream

    @classmethod
    def open_pid_specific_error_stream(cls, path):
        ret = safe_open(path, mode="w")

        def unlink_if_empty():
            try:
                if os.path.getsize(path) == 0:
                    os.unlink(path)
            except OSError:
                pass

        # NB: This will only get called if nothing fatal happens, but that's precisely when we want
        # to get called. If anything fatal happens there should be an exception written to the log,
        # and therefore we don't want to unlink it.
        atexit.register(unlink_if_empty)
        return ret

    @classmethod
    def exceptions_log_path(cls, for_pid=None, in_dir=None):
        """Get the path to either the shared or pid-specific fatal errors log file."""
        if for_pid is None:
            intermediate_filename_component = ""
        else:
            assert isinstance(for_pid, Pid)
            intermediate_filename_component = f".{for_pid}"
        in_dir = in_dir or cls._log_dir
        return os.path.join(in_dir, f"exceptions{intermediate_filename_component}.log")

    @classmethod
    def _log_exception(cls, msg):
        """Try to log an error message to this process's error log and the shared error log.

        NB: Doesn't raise (logs an error instead).
        """
        pid = os.getpid()
        fatal_error_log_entry = cls._format_exception_message(msg, pid)

        # We care more about this log than the shared log, so write to it first.
        try:
            cls._try_write_with_flush(cls._pid_specific_error_fileobj, fatal_error_log_entry)
        except Exception as e:
            logger.error(
                f"Error logging the message '{msg}' to the pid-specific file handle for {cls._log_dir} at pid {pid}:\n{e}"
            )

        # Write to the shared log.
        try:
            # TODO: we should probably guard this against concurrent modification by other pants
            # subprocesses somehow.
            cls._try_write_with_flush(cls._shared_error_fileobj, fatal_error_log_entry)
        except Exception as e:
            logger.error(
                f"Error logging the message '{msg}' to the shared file handle for {cls._log_dir} at pid {pid}:\n{e}"
            )

    @classmethod
    def _try_write_with_flush(cls, fileobj, payload):
        """This method is here so that it can be patched to simulate write errors.

        This is because mock can't patch primitive objects like file objects.
        """
        fileobj.write(payload)
        fileobj.flush()

    @classmethod
    def reset_signal_handler(cls, signal_handler: SignalHandler) -> SignalHandler:
        """Given a SignalHandler, uses the `signal` std library functionality to set the pants
        process's signal handlers to those specified in the object.

        Note that since this calls `signal.signal()`, it will crash if not the main thread. Returns
        the previously-registered signal handler.
        """

        for signum, handler in signal_handler.signal_handler_mapping.items():
            signal.signal(signum, handler)
            # Retry any system calls interrupted by any of the signals we just installed handlers for
            # (instead of having them raise EINTR). siginterrupt(3) says this is the default behavior on
            # Linux and OSX.
            signal.siginterrupt(signum, False)

        previous_signal_handler = cls._signal_handler
        cls._signal_handler = signal_handler

        return previous_signal_handler

    @classmethod
    @contextmanager
    def trapped_signals(cls, new_signal_handler: SignalHandler) -> Iterator[None]:
        """A contextmanager which temporarily overrides signal handling.

        NB: This method calls signal.signal(), which will crash if not called from the main thread!
        """
        previous_signal_handler = cls.reset_signal_handler(new_signal_handler)
        try:
            yield
        finally:
            cls.reset_signal_handler(previous_signal_handler)

    @classmethod
    @contextmanager
    def ignoring_sigint(cls) -> Iterator[None]:
        """This method provides a context that temporarily disables responding to the SIGINT signal
        sent by a Ctrl-C in the terminal.

        We currently only use this to implement disabling catching SIGINT while an
        InteractiveProcess is running (where we want that process to catch it), and only when pantsd
        is not enabled. If pantsd is enabled, the client will actually catch SIGINT and forward it
        to the server, so we don't want the server process to ignore it.
        """

        try:
            cls._signal_handler._toggle_ignoring_sigint(True)
            yield
        finally:
            cls._signal_handler._toggle_ignoring_sigint(False)

    @classmethod
    def _iso_timestamp_for_now(cls):
        return datetime.datetime.now().isoformat()

    # NB: This includes a trailing newline, but no leading newline.
    _EXCEPTION_LOG_FORMAT = """\
timestamp: {timestamp}
process title: {process_title}
sys.argv: {args}
pid: {pid}
{message}
"""

    @classmethod
    def _format_exception_message(cls, msg, pid):
        return cls._EXCEPTION_LOG_FORMAT.format(
            timestamp=cls._iso_timestamp_for_now(),
            process_title=setproctitle.getproctitle(),
            args=sys.argv,
            pid=pid,
            message=msg,
        )

    _traceback_omitted_default_text = "(backtrace omitted)"

    @classmethod
    def _format_traceback(cls, traceback_lines, should_print_backtrace):
        if should_print_backtrace:
            traceback_string = "\n{}".format("".join(traceback_lines))
        else:
            traceback_string = f" {cls._traceback_omitted_default_text}"
        return traceback_string

    _UNHANDLED_EXCEPTION_LOG_FORMAT = """\
Exception caught: ({exception_type}){backtrace}
Exception message: {exception_message}{maybe_newline}
"""

    @classmethod
    def _format_unhandled_exception_log(cls, exc, tb, add_newline, should_print_backtrace):
        exc_type = type(exc)
        exception_full_name = f"{exc_type.__module__}.{exc_type.__name__}"
        exception_message = str(exc) if exc else "(no message)"
        maybe_newline = "\n" if add_newline else ""
        return cls._UNHANDLED_EXCEPTION_LOG_FORMAT.format(
            exception_type=exception_full_name,
            backtrace=cls._format_traceback(
                traceback_lines=traceback.format_tb(tb),
                should_print_backtrace=should_print_backtrace,
            ),
            exception_message=exception_message,
            maybe_newline=maybe_newline,
        )

    @classmethod
    def log_exception(cls, exc_class=None, exc=None, tb=None, add_newline=False):
        """Logs an unhandled exception to a variety of locations."""
        exc_class = exc_class or sys.exc_info()[0]
        exc = exc or sys.exc_info()[1]
        tb = tb or sys.exc_info()[2]

        # This exception was raised by a signal handler with the intent to exit the program.
        if exc_class == SignalHandler.SignalHandledNonLocalExit:
            return cls._handle_signal_gracefully(exc.signum, exc.signame, exc.traceback_lines)

        extra_err_msg = None
        try:
            # Always output the unhandled exception details into a log file, including the
            # traceback.
            exception_log_entry = cls._format_unhandled_exception_log(
                exc, tb, add_newline, should_print_backtrace=True
            )
            cls._log_exception(exception_log_entry)
        except Exception as e:
            extra_err_msg = f"Additional error logging unhandled exception {exc}: {e}"
            logger.error(extra_err_msg)

        # The rust logger implementation will have its own stacktrace, but at import time, we want
        # to be able to see any stacktrace to know where the error is being raised, so we reproduce
        # it here.
        exception_log_entry = cls._format_unhandled_exception_log(
            exc, tb, add_newline, should_print_backtrace=True
        )
        logger.exception(exception_log_entry)

    @classmethod
    def _handle_signal_gracefully(cls, signum, signame, traceback_lines):
        """Signal handler for non-fatal signals which raises or logs an error."""

        def gen_formatted(formatted_traceback: str) -> str:
            return f"Signal {signum} ({signame}) was raised. Exiting with failure.{formatted_traceback}"

        # Extract the stack, and format an entry to be written to the exception log.
        formatted_traceback = cls._format_traceback(
            traceback_lines=traceback_lines, should_print_backtrace=True
        )

        signal_error_log_entry = gen_formatted(formatted_traceback)

        # TODO: determine the appropriate signal-safe behavior here (to avoid writing to our file
        # descriptors reentrantly, which raises an IOError).
        # This method catches any exceptions raised within it.
        cls._log_exception(signal_error_log_entry)

        # Create a potentially-abbreviated traceback for the terminal or other interactive stream.
        formatted_traceback_for_terminal = cls._format_traceback(
            traceback_lines=traceback_lines,
            should_print_backtrace=True,
        )

        terminal_log_entry = gen_formatted(formatted_traceback_for_terminal)

        # Print the output via standard logging.
        logger.error(terminal_log_entry)

1	# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	import atexit	2✔
5	import datetime	2✔
6	import faulthandler	2✔
7	import logging	2✔
8	import os	2✔
9	import signal	2✔
10	import sys	2✔
11	import threading	2✔
12	import traceback	2✔
13	from collections.abc import Callable, Iterator	2✔
14	from contextlib import contextmanager	2✔
15
16	import psutil	2✔
17	import setproctitle	2✔
18
19	from pants.util.dirutil import safe_mkdir, safe_open	2✔
20	from pants.util.osutil import Pid	2✔
21
22	logger = logging.getLogger(__name__)	2✔
23
24
25	class SignalHandler:	2✔
26	"""A specification for how to handle a fixed set of nonfatal signals.
27
28	This is subclassed and registered with ExceptionSink.reset_signal_handler() whenever the signal
29	handling behavior is modified for different pants processes, for example in the remote client when
30	pantsd is enabled. The default behavior is to exit "gracefully" by leaving a detailed log of which
31	signal was received, then exiting with failure.
32
33	Note that the terminal converts a ctrl-c from the user into a SIGINT.
34	"""
35
36	@property	2✔
37	def signal_handler_mapping(self) -> dict[signal.Signals, Callable]:	2✔
38	"""A dict mapping (signal number) -> (a method handling the signal)."""
39	# Could use an enum here, but we never end up doing any matching on the specific signal value,
40	# instead just iterating over the registered signals to set handlers, so a dict is probably
41	# better.
42	return {	×
43	signal.SIGINT: self._handle_sigint_if_enabled,
44	signal.SIGQUIT: self.handle_sigquit,
45	signal.SIGTERM: self.handle_sigterm,
46	}
47
48	def __init__(self, *, pantsd_instance: bool):	2✔
49	self._ignore_sigint_lock = threading.Lock()	2✔
50	self._ignoring_sigint = False	2✔
51	self._pantsd_instance = pantsd_instance	2✔
52
53	def _handle_sigint_if_enabled(self, signum: int, _frame):	2✔
54	with self._ignore_sigint_lock:	×
55	if not self._ignoring_sigint:	×
56	self.handle_sigint(signum, _frame)	×
57
58	def _toggle_ignoring_sigint(self, toggle: bool) -> None:	2✔
59	if not self._pantsd_instance:	×
60	with self._ignore_sigint_lock:	×
61	self._ignoring_sigint = toggle	×
62
63	def _send_signal_to_children(self, received_signal: int, signame: str) -> None:	2✔
64	"""Send a signal to any children of this process in order.
65
66	Pants may have spawned multiple subprocesses via Python or Rust. Upon receiving a signal,
67	this method is invoked to propagate the signal to all children, regardless of how they were
68	spawned.
69	"""
70
71	self_process = psutil.Process()	×
72	children = self_process.children()	×
73	logger.debug(f"Sending signal {signame} ({received_signal}) to child processes: {children}")	×
74	for child_process in children:	×
75	child_process.send_signal(received_signal)	×
76
77	def handle_sigint(self, signum: int, _frame):	2✔
78	self._send_signal_to_children(signum, "SIGINT")	×
79	raise KeyboardInterrupt("User interrupted execution with control-c!")	×
80
81	# TODO(#7406): figure out how to let sys.exit work in a signal handler instead of having to raise
82	# this exception!
83	class SignalHandledNonLocalExit(Exception):	2✔
84	"""Raised in handlers for non-fatal signals to overcome Python limitations.
85
86	When waiting on a subprocess and in a signal handler, sys.exit appears to be ignored, and
87	causes the signal handler to return. We want to (eventually) exit after these signals, not
88	ignore them, so we raise this exception instead and check it in our sys.excepthook override.
89	"""
90
91	def __init__(self, signum, signame):	2✔
92	self.signum = signum	×
93	self.signame = signame	×
94	self.traceback_lines = traceback.format_stack()	×
95	super().__init__()	×
96
97	if "I/O operation on closed file" in self.traceback_lines:	×
98	logger.debug(	×
99	"SignalHandledNonLocalExit: unexpected appearance of "
100	"'I/O operation on closed file' in traceback"
101	)
102
103	def handle_sigquit(self, signum, _frame):	2✔
104	self._send_signal_to_children(signum, "SIGQUIT")	×
105	raise self.SignalHandledNonLocalExit(signum, "SIGQUIT")	×
106
107	def handle_sigterm(self, signum, _frame):	2✔
108	self._send_signal_to_children(signum, "SIGTERM")	×
109	raise self.SignalHandledNonLocalExit(signum, "SIGTERM")	×
110
111
112	class ExceptionSink:	2✔
113	"""A mutable singleton object representing where exceptions should be logged to.
114
115	The ExceptionSink should be installed in any process that is running Pants @rules via the
116	engine. Notably, this does _not_ include the pantsd client, which does its own signal handling
117	directly in order to forward information to the pantsd server.
118	"""
119
120	# NB: see the bottom of this file where we call reset_log_location() and other mutators in order
121	# to properly setup global state.
122	_log_dir = None	2✔
123
124	# Where to log stacktraces to in a SIGUSR2 handler.
125	_interactive_output_stream = None	2✔
126
127	# An instance of `SignalHandler` which is invoked to handle a static set of specific nonfatal
128	# signals (these signal handlers are allowed to make pants exit, but unlike SIGSEGV they don't
129	# need to exit immediately).
130	_signal_handler: SignalHandler = SignalHandler(pantsd_instance=False)	2✔
131
132	# These persistent open file descriptors are kept so the signal handler can do almost no work
133	# (and lets faulthandler figure out signal safety).
134	_pid_specific_error_fileobj = None	2✔
135	_shared_error_fileobj = None	2✔
136
137	def __new__(cls, args, *kwargs):	2✔
138	raise TypeError(	×
139	f"Instances of {cls.__name__} are not allowed to be constructed! Call install() instead."
140	)
141
142	class ExceptionSinkError(Exception):	2✔
143	pass	2✔
144
145	@classmethod	2✔
146	def install(cls, log_location: str, pantsd_instance: bool) -> None:	2✔
147	"""Setup global state for this process, such as signal handlers and sys.excepthook."""
148
149	# Set the log location for writing logs before bootstrap options are parsed.
150	cls.reset_log_location(log_location)	×
151
152	# NB: Mutate process-global state!
153	sys.excepthook = ExceptionSink.log_exception	×
154
155	# Setup a default signal handler.
156	cls.reset_signal_handler(SignalHandler(pantsd_instance=pantsd_instance))	×
157
158	# All reset_* methods are ~idempotent!
159	@classmethod	2✔
160	def reset_log_location(cls, new_log_location: str) -> None:	2✔
161	"""Re-acquire file handles to error logs based in the new location.
162
163	Class state:
164	- Overwrites `cls._log_dir`, `cls._pid_specific_error_fileobj`, and
165	`cls._shared_error_fileobj`.
166	OS state:
167	- May create a new directory.
168	- Overwrites signal handlers for many fatal and non-fatal signals (but not SIGUSR2).
169
170	:raises: :class:`ExceptionSink.ExceptionSinkError` if the directory does not exist or is not
171	writable.
172	"""
173	# We could no-op here if the log locations are the same, but there's no reason not to have the
174	# additional safety of re-acquiring file descriptors each time (and erroring out early if the
175	# location is no longer writable).
176	try:	×
177	safe_mkdir(new_log_location)	×
178	except Exception as e:	×
179	raise cls.ExceptionSinkError(	×
180	f"The provided log location path at '{new_log_location}' is not writable or could not be created: {str(e)}.",
181	e,
182	)
183
184	pid = os.getpid()	×
185	pid_specific_log_path = cls.exceptions_log_path(for_pid=pid, in_dir=new_log_location)	×
186	shared_log_path = cls.exceptions_log_path(in_dir=new_log_location)	×
187	assert pid_specific_log_path != shared_log_path	×
188	try:	×
189	pid_specific_error_stream = cls.open_pid_specific_error_stream(pid_specific_log_path)	×
190	shared_error_stream = safe_open(shared_log_path, mode="a")	×
191	except Exception as e:	×
192	raise cls.ExceptionSinkError(	×
193	f"Error opening fatal error log streams for log location '{new_log_location}': {str(e)}"
194	)
195
196	# NB: mutate process-global state!
197	if faulthandler.is_enabled():	×
198	logger.debug("re-enabling faulthandler")	×
199	# Call Py_CLEAR() on the previous error stream:
200	# https://github.com/vstinner/faulthandler/blob/master/faulthandler.c
201	faulthandler.disable()	×
202	# Send a stacktrace to this file if interrupted by a fatal error.
203	faulthandler.enable(file=pid_specific_error_stream, all_threads=True)	×
204
205	# NB: mutate the class variables!
206	cls._log_dir = new_log_location	×
207	cls._pid_specific_error_fileobj = pid_specific_error_stream	×
208	cls._shared_error_fileobj = shared_error_stream	×
209
210	@classmethod	2✔
211	def open_pid_specific_error_stream(cls, path):	2✔
212	ret = safe_open(path, mode="w")	×
213
214	def unlink_if_empty():	×
215	try:	×
216	if os.path.getsize(path) == 0:	×
217	os.unlink(path)	×
218	except OSError:	×
219	pass	×
220
221	# NB: This will only get called if nothing fatal happens, but that's precisely when we want
222	# to get called. If anything fatal happens there should be an exception written to the log,
223	# and therefore we don't want to unlink it.
224	atexit.register(unlink_if_empty)	×
225	return ret	×
226
227	@classmethod	2✔
228	def exceptions_log_path(cls, for_pid=None, in_dir=None):	2✔
229	"""Get the path to either the shared or pid-specific fatal errors log file."""
230	if for_pid is None:	×
231	intermediate_filename_component = ""	×
232	else:
233	assert isinstance(for_pid, Pid)	×
234	intermediate_filename_component = f".{for_pid}"	×
235	in_dir = in_dir or cls._log_dir	×
236	return os.path.join(in_dir, f"exceptions{intermediate_filename_component}.log")	×
237
238	@classmethod	2✔
239	def _log_exception(cls, msg):	2✔
240	"""Try to log an error message to this process's error log and the shared error log.
241
242	NB: Doesn't raise (logs an error instead).
243	"""
244	pid = os.getpid()	×
245	fatal_error_log_entry = cls._format_exception_message(msg, pid)	×
246
247	# We care more about this log than the shared log, so write to it first.
248	try:	×
249	cls._try_write_with_flush(cls._pid_specific_error_fileobj, fatal_error_log_entry)	×
250	except Exception as e:	×
251	logger.error(	×
252	f"Error logging the message '{msg}' to the pid-specific file handle for {cls._log_dir} at pid {pid}:\n{e}"
253	)
254
255	# Write to the shared log.
256	try:	×
257	# TODO: we should probably guard this against concurrent modification by other pants
258	# subprocesses somehow.
259	cls._try_write_with_flush(cls._shared_error_fileobj, fatal_error_log_entry)	×
260	except Exception as e:	×
261	logger.error(	×
262	f"Error logging the message '{msg}' to the shared file handle for {cls._log_dir} at pid {pid}:\n{e}"
263	)
264
265	@classmethod	2✔
266	def _try_write_with_flush(cls, fileobj, payload):	2✔
267	"""This method is here so that it can be patched to simulate write errors.
268
269	This is because mock can't patch primitive objects like file objects.
270	"""
271	fileobj.write(payload)	×
272	fileobj.flush()	×
273
274	@classmethod	2✔
275	def reset_signal_handler(cls, signal_handler: SignalHandler) -> SignalHandler:	2✔
276	"""Given a SignalHandler, uses the `signal` std library functionality to set the pants
277	process's signal handlers to those specified in the object.
278
279	Note that since this calls `signal.signal()`, it will crash if not the main thread. Returns
280	the previously-registered signal handler.
281	"""
282
283	for signum, handler in signal_handler.signal_handler_mapping.items():	×
284	signal.signal(signum, handler)	×
285	# Retry any system calls interrupted by any of the signals we just installed handlers for
286	# (instead of having them raise EINTR). siginterrupt(3) says this is the default behavior on
287	# Linux and OSX.
288	signal.siginterrupt(signum, False)	×
289
290	previous_signal_handler = cls._signal_handler	×
291	cls._signal_handler = signal_handler	×
292
293	return previous_signal_handler	×
294
295	@classmethod	2✔
296	@contextmanager	2✔
297	def trapped_signals(cls, new_signal_handler: SignalHandler) -> Iterator[None]:	2✔
298	"""A contextmanager which temporarily overrides signal handling.
299
300	NB: This method calls signal.signal(), which will crash if not called from the main thread!
301	"""
302	previous_signal_handler = cls.reset_signal_handler(new_signal_handler)	×
303	try:	×
304	yield	×
305	finally:
306	cls.reset_signal_handler(previous_signal_handler)	×
307
308	@classmethod	2✔
309	@contextmanager	2✔
310	def ignoring_sigint(cls) -> Iterator[None]:	2✔
311	"""This method provides a context that temporarily disables responding to the SIGINT signal
312	sent by a Ctrl-C in the terminal.
313
314	We currently only use this to implement disabling catching SIGINT while an
315	InteractiveProcess is running (where we want that process to catch it), and only when pantsd
316	is not enabled. If pantsd is enabled, the client will actually catch SIGINT and forward it
317	to the server, so we don't want the server process to ignore it.
318	"""
319
320	try:	×
321	cls._signal_handler._toggle_ignoring_sigint(True)	×
322	yield	×
323	finally:
324	cls._signal_handler._toggle_ignoring_sigint(False)	×
325
326	@classmethod	2✔
327	def _iso_timestamp_for_now(cls):	2✔
328	return datetime.datetime.now().isoformat()	×
329
330	# NB: This includes a trailing newline, but no leading newline.
331	_EXCEPTION_LOG_FORMAT = """\	2✔
332	timestamp: {timestamp}
333	process title: {process_title}
334	sys.argv: {args}
335	pid: {pid}
336	{message}
337	"""
338
339	@classmethod	2✔
340	def _format_exception_message(cls, msg, pid):	2✔
341	return cls._EXCEPTION_LOG_FORMAT.format(	×
342	timestamp=cls._iso_timestamp_for_now(),
343	process_title=setproctitle.getproctitle(),
344	args=sys.argv,
345	pid=pid,
346	message=msg,
347	)
348
349	_traceback_omitted_default_text = "(backtrace omitted)"	2✔
350
351	@classmethod	2✔
352	def _format_traceback(cls, traceback_lines, should_print_backtrace):	2✔
353	if should_print_backtrace:	×
354	traceback_string = "\n{}".format("".join(traceback_lines))	×
355	else:
356	traceback_string = f" {cls._traceback_omitted_default_text}"	×
357	return traceback_string	×
358
359	_UNHANDLED_EXCEPTION_LOG_FORMAT = """\	2✔
360	Exception caught: ({exception_type}){backtrace}
361	Exception message: {exception_message}{maybe_newline}
362	"""
363
364	@classmethod	2✔
365	def _format_unhandled_exception_log(cls, exc, tb, add_newline, should_print_backtrace):	2✔
366	exc_type = type(exc)	×
367	exception_full_name = f"{exc_type.__module__}.{exc_type.__name__}"	×
368	exception_message = str(exc) if exc else "(no message)"	×
369	maybe_newline = "\n" if add_newline else ""	×
370	return cls._UNHANDLED_EXCEPTION_LOG_FORMAT.format(	×
371	exception_type=exception_full_name,
372	backtrace=cls._format_traceback(
373	traceback_lines=traceback.format_tb(tb),
374	should_print_backtrace=should_print_backtrace,
375	),
376	exception_message=exception_message,
377	maybe_newline=maybe_newline,
378	)
379
380	@classmethod	2✔
381	def log_exception(cls, exc_class=None, exc=None, tb=None, add_newline=False):	2✔
382	"""Logs an unhandled exception to a variety of locations."""
383	exc_class = exc_class or sys.exc_info()[0]	×
384	exc = exc or sys.exc_info()[1]	×
385	tb = tb or sys.exc_info()[2]	×
386
387	# This exception was raised by a signal handler with the intent to exit the program.
388	if exc_class == SignalHandler.SignalHandledNonLocalExit:	×
389	return cls._handle_signal_gracefully(exc.signum, exc.signame, exc.traceback_lines)	×
390
391	extra_err_msg = None	×
392	try:	×
393	# Always output the unhandled exception details into a log file, including the
394	# traceback.
395	exception_log_entry = cls._format_unhandled_exception_log(	×
396	exc, tb, add_newline, should_print_backtrace=True
397	)
398	cls._log_exception(exception_log_entry)	×
399	except Exception as e:	×
400	extra_err_msg = f"Additional error logging unhandled exception {exc}: {e}"	×
401	logger.error(extra_err_msg)	×
402
403	# The rust logger implementation will have its own stacktrace, but at import time, we want
404	# to be able to see any stacktrace to know where the error is being raised, so we reproduce
405	# it here.
406	exception_log_entry = cls._format_unhandled_exception_log(	×
407	exc, tb, add_newline, should_print_backtrace=True
408	)
409	logger.exception(exception_log_entry)	×
410
411	@classmethod	2✔
412	def _handle_signal_gracefully(cls, signum, signame, traceback_lines):	2✔
413	"""Signal handler for non-fatal signals which raises or logs an error."""
414
415	def gen_formatted(formatted_traceback: str) -> str:	×
416	return f"Signal {signum} ({signame}) was raised. Exiting with failure.{formatted_traceback}"	×
417
418	# Extract the stack, and format an entry to be written to the exception log.
419	formatted_traceback = cls._format_traceback(	×
420	traceback_lines=traceback_lines, should_print_backtrace=True
421	)
422
423	signal_error_log_entry = gen_formatted(formatted_traceback)	×
424
425	# TODO: determine the appropriate signal-safe behavior here (to avoid writing to our file
426	# descriptors reentrantly, which raises an IOError).
427	# This method catches any exceptions raised within it.
428	cls._log_exception(signal_error_log_entry)	×
429
430	# Create a potentially-abbreviated traceback for the terminal or other interactive stream.
431	formatted_traceback_for_terminal = cls._format_traceback(	×
432	traceback_lines=traceback_lines,
433	should_print_backtrace=True,
434	)
435
436	terminal_log_entry = gen_formatted(formatted_traceback_for_terminal)	×
437
438	# Print the output via standard logging.
439	logger.error(terminal_log_entry)	×

pantsbuild / pants / 20328535594

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous