26342152999

Committed 23 May 2026 07:59PM UTC coverage: 91.165% (-1.6%) from 92.792%

Build # 26342152999

Build Type

push

github

Committed by

web-flow

Commit Message

Run Linux ARM CI on Depot runners (#23363)

RunsOn is deprecating their v2 stack, and rather than migrate
to v3 we should use the resources kindly donated by Depot.

GitHub also now has Linux ARM runners, should we need them.

Coverage Stats

87305 of 95766 relevant lines covered (91.16%)

3.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.49

/src/python/pants/pantsd/pantsd_integration_test_base.py

# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import functools
import os
import time
import unittest
from collections.abc import Callable, Iterator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Any

from colors import bold, cyan, magenta

from pants.pantsd.process_manager import ProcessManager
from pants.testutil.pants_integration_test import (
    PantsJoinHandle,
    PantsResult,
    kill_daemon,
    read_pants_log,
    run_pants,
    run_pants_with_workdir,
    run_pants_with_workdir_without_waiting,
)
from pants.util.collections import recursively_update
from pants.util.contextutil import temporary_dir
from pants.util.dirutil import maybe_read_file


def banner(s):
    print(cyan("=" * 63))
    print(cyan(f"- {s} {('-' * (60 - len(s)))}"))
    print(cyan("=" * 63))


def attempts(
    msg: str,
    *,
    delay: float = 0.5,
    timeout: float = 30,
    backoff: float = 1.2,
) -> Iterator[None]:
    """A generator that yields a number of times before failing.

    A caller should break out of a loop on the generator in order to succeed.
    """
    count = 0
    deadline = time.time() + timeout
    while time.time() < deadline:
        count += 1
        yield
        time.sleep(delay)
        delay *= backoff
    raise AssertionError(f"After {count} attempts in {timeout} seconds: {msg}")


def launch_waiter(
    *, workdir: str, config: Mapping | None = None, cleanup_wait_time: int = 0
) -> tuple[PantsJoinHandle, int, int, str]:
    """Launch a process that will wait forever for a file to be created.

    Returns the pants client handle, the pid of the waiting process, the pid of a child of the
    waiting process, and the file to create to cause the waiting child to exit.
    """
    file_to_make = os.path.join(workdir, "some_magic_file")
    waiter_pid_file = os.path.join(workdir, "pid_file")
    child_pid_file = os.path.join(workdir, "child_pid_file")

    argv = [
        "run",
        "testprojects/src/python/coordinated_runs:waiter",
        "--",
        file_to_make,
        waiter_pid_file,
        child_pid_file,
        str(cleanup_wait_time),
    ]
    client_handle = run_pants_with_workdir_without_waiting(argv, workdir=workdir, config=config)
    waiter_pid = -1
    for _ in attempts("The waiter process should have written its pid."):
        waiter_pid_str = maybe_read_file(waiter_pid_file)
        child_pid_str = maybe_read_file(child_pid_file)
        if waiter_pid_str and child_pid_str:
            waiter_pid = int(waiter_pid_str)
            child_pid = int(child_pid_str)
            break
    return client_handle, waiter_pid, child_pid, file_to_make


class PantsDaemonMonitor(ProcessManager):
    def __init__(self, metadata_base_dir: str):
        super().__init__(name="pantsd", metadata_base_dir=metadata_base_dir)
        self._started = False

    def _log(self):
        print(magenta(f"PantsDaemonMonitor: pid is {self.pid} is_alive={self.is_alive()}"))

    def assert_started_and_stopped(self, timeout: int = 30) -> None:
        """Asserts that pantsd was alive (it wrote a pid file), but that it stops afterward."""
        self.await_pid(timeout)
        self._started = True
        self.assert_stopped()

    def assert_started(self, timeout=30):
        self.await_pid(timeout)
        self._started = True
        self._check_pantsd_is_alive()
        return self.pid

    def _check_pantsd_is_alive(self):
        self._log()
        assert self._started, (
            "cannot assert that pantsd is running. Try calling assert_started before calling this method."
        )
        assert self.is_alive(), "pantsd was not alive."
        return self.pid

    def current_memory_usage(self):
        """Return the current memory usage of the pantsd process (which must be running)

        :return: memory usage in bytes
        """
        self.assert_running()
        return self._as_process().memory_info()[0]

    def assert_running(self):
        if not self._started:
            return self.assert_started()
        else:
            return self._check_pantsd_is_alive()

    def assert_stopped(self):
        self._log()
        assert self._started, (
            "cannot assert pantsd stoppage. Try calling assert_started before calling this method."
        )
        for _ in attempts("pantsd should be stopped!"):
            if self.is_dead():
                break


@dataclass(frozen=True)
class PantsdRunContext:
    runner: Callable[..., Any]
    checker: PantsDaemonMonitor
    workdir: str
    pantsd_config: dict[str, Any]


class PantsDaemonIntegrationTestBase(unittest.TestCase):
    @staticmethod
    def run_pants(*args, **kwargs) -> PantsResult:
        # We set our own ad-hoc pantsd configuration in most of these tests.
        return run_pants(*args, **{**kwargs, **{"use_pantsd": False}})

    @staticmethod
    def run_pants_with_workdir(*args, **kwargs) -> PantsResult:
        # We set our own ad-hoc pantsd configuration in most of these tests.
        return run_pants_with_workdir(*args, **{**kwargs, **{"use_pantsd": False}})

    @staticmethod
    def run_pants_with_workdir_without_waiting(*args, **kwargs) -> PantsJoinHandle:
        # We set our own ad-hoc pantsd configuration in most of these tests.
        return run_pants_with_workdir_without_waiting(*args, **{**kwargs, **{"use_pantsd": False}})

    @contextmanager
    def pantsd_test_context(
        self, *, log_level: str = "info", extra_config: dict[str, Any] | None = None
    ) -> Iterator[tuple[str, dict[str, Any], PantsDaemonMonitor]]:
        with temporary_dir(root_dir=os.getcwd()) as dot_pants_dot_d:
            pid_dir = os.path.join(dot_pants_dot_d, "pids")
            workdir = os.path.join(dot_pants_dot_d, "workdir")
            print(f"\npantsd log is {workdir}/pants.log")
            pantsd_config = {
                "GLOBAL": {
                    "pantsd": True,
                    "level": log_level,
                    "pants_subprocessdir": pid_dir,
                    "backend_packages": [
                        # Provide goals used by various tests.
                        "pants.backend.python",
                        "pants.backend.python.lint.flake8",
                    ],
                },
                "python": {
                    "interpreter_constraints": "['>=3.14,<3.15']",
                },
            }

            if extra_config:
                recursively_update(pantsd_config, extra_config)
            print(f">>> config: \n{pantsd_config}\n")

            checker = PantsDaemonMonitor(pid_dir)
            kill_daemon(pid_dir)
            try:
                yield workdir, pantsd_config, checker
                kill_daemon(pid_dir)
                checker.assert_stopped()
            finally:
                banner("BEGIN pants.log")
                for line in read_pants_log(workdir):
                    print(line)
                banner("END pants.log")

    @contextmanager
    def pantsd_successful_run_context(self, *args, **kwargs) -> Iterator[PantsdRunContext]:
        with self.pantsd_run_context(*args, success=True, **kwargs) as context:  # type: ignore[misc]
            yield context

    @contextmanager
    def pantsd_run_context(
        self,
        log_level: str = "info",
        extra_config: dict[str, Any] | None = None,
        extra_env: dict[str, str] | None = None,
        success: bool = True,
    ) -> Iterator[PantsdRunContext]:
        with self.pantsd_test_context(log_level=log_level, extra_config=extra_config) as (
            workdir,
            pantsd_config,
            checker,
        ):
            runner = functools.partial(
                self.assert_runner,
                workdir,
                pantsd_config,
                extra_env=extra_env,
                success=success,
            )
            yield PantsdRunContext(
                runner=runner, checker=checker, workdir=workdir, pantsd_config=pantsd_config
            )

    def _run_count(self, workdir):
        run_tracker_dir = os.path.join(workdir, "run-tracker")
        if os.path.isdir(run_tracker_dir):
            return len([f for f in os.listdir(run_tracker_dir) if f != "latest"])
        else:
            return 0

    def assert_runner(
        self,
        workdir: str,
        config,
        cmd,
        extra_config=None,
        extra_env=None,
        success=True,
        expected_runs: int = 1,
    ):
        combined_config = config.copy()
        recursively_update(combined_config, extra_config or {})
        print(
            bold(
                cyan(
                    "\nrunning: ./pants {} (config={}) (extra_env={})".format(
                        " ".join(cmd), combined_config, extra_env
                    )
                )
            )
        )
        run_count = self._run_count(workdir)
        start_time = time.time()
        run = self.run_pants_with_workdir(
            cmd, workdir=workdir, config=combined_config, extra_env=extra_env or {}
        )
        elapsed = time.time() - start_time
        print(bold(cyan(f"\ncompleted in {elapsed} seconds")))

        if success:
            run.assert_success()
        else:
            run.assert_failure()

        runs_created = self._run_count(workdir) - run_count
        self.assertEqual(
            runs_created,
            expected_runs,
            f"Expected {expected_runs} RunTracker run(s) to be created per pantsd run: was {runs_created}",
        )

        return run

1	# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	5✔
5
6	import functools	5✔
7	import os	5✔
8	import time	5✔
9	import unittest	5✔
10	from collections.abc import Callable, Iterator, Mapping	5✔
11	from contextlib import contextmanager	5✔
12	from dataclasses import dataclass	5✔
13	from typing import Any	5✔
14
15	from colors import bold, cyan, magenta	5✔
16
17	from pants.pantsd.process_manager import ProcessManager	5✔
18	from pants.testutil.pants_integration_test import (	5✔
19	PantsJoinHandle,
20	PantsResult,
21	kill_daemon,
22	read_pants_log,
23	run_pants,
24	run_pants_with_workdir,
25	run_pants_with_workdir_without_waiting,
26	)
27	from pants.util.collections import recursively_update	5✔
28	from pants.util.contextutil import temporary_dir	5✔
29	from pants.util.dirutil import maybe_read_file	5✔
30
31
32	def banner(s):	5✔
33	print(cyan("=" * 63))	5✔
34	print(cyan(f"- {s} {('-' * (60 - len(s)))}"))	5✔
35	print(cyan("=" * 63))	5✔
36
37
38	def attempts(	5✔
39	msg: str,
40	*,
41	delay: float = 0.5,
42	timeout: float = 30,
43	backoff: float = 1.2,
44	) -> Iterator[None]:
45	"""A generator that yields a number of times before failing.
46
47	A caller should break out of a loop on the generator in order to succeed.
48	"""
49	count = 0	5✔
50	deadline = time.time() + timeout	5✔
51	while time.time() < deadline:	5✔
52	count += 1	5✔
53	yield	5✔
54	time.sleep(delay)	1✔
55	delay *= backoff	1✔
56	raise AssertionError(f"After {count} attempts in {timeout} seconds: {msg}")	×
57
58
59	def launch_waiter(	5✔
60	*, workdir: str, config: Mapping \| None = None, cleanup_wait_time: int = 0
61	) -> tuple[PantsJoinHandle, int, int, str]:
62	"""Launch a process that will wait forever for a file to be created.
63
64	Returns the pants client handle, the pid of the waiting process, the pid of a child of the
65	waiting process, and the file to create to cause the waiting child to exit.
66	"""
67	file_to_make = os.path.join(workdir, "some_magic_file")	1✔
68	waiter_pid_file = os.path.join(workdir, "pid_file")	1✔
69	child_pid_file = os.path.join(workdir, "child_pid_file")	1✔
70
71	argv = [	1✔
72	"run",
73	"testprojects/src/python/coordinated_runs:waiter",
74	"--",
75	file_to_make,
76	waiter_pid_file,
77	child_pid_file,
78	str(cleanup_wait_time),
79	]
80	client_handle = run_pants_with_workdir_without_waiting(argv, workdir=workdir, config=config)	1✔
81	waiter_pid = -1	1✔
82	for _ in attempts("The waiter process should have written its pid."):	1✔
83	waiter_pid_str = maybe_read_file(waiter_pid_file)	1✔
84	child_pid_str = maybe_read_file(child_pid_file)	1✔
85	if waiter_pid_str and child_pid_str:	1✔
86	waiter_pid = int(waiter_pid_str)	1✔
87	child_pid = int(child_pid_str)	1✔
88	break	1✔
89	return client_handle, waiter_pid, child_pid, file_to_make	1✔
90
91
92	class PantsDaemonMonitor(ProcessManager):	5✔
93	def __init__(self, metadata_base_dir: str):	5✔
94	super().__init__(name="pantsd", metadata_base_dir=metadata_base_dir)	5✔
95	self._started = False	5✔
96
97	def _log(self):	5✔
98	print(magenta(f"PantsDaemonMonitor: pid is {self.pid} is_alive={self.is_alive()}"))	5✔
99
100	def assert_started_and_stopped(self, timeout: int = 30) -> None:	5✔
101	"""Asserts that pantsd was alive (it wrote a pid file), but that it stops afterward."""
102	self.await_pid(timeout)	×
103	self._started = True	×
104	self.assert_stopped()	×
105
106	def assert_started(self, timeout=30):	5✔
107	self.await_pid(timeout)	5✔
108	self._started = True	5✔
109	self._check_pantsd_is_alive()	5✔
110	return self.pid	5✔
111
112	def _check_pantsd_is_alive(self):	5✔
113	self._log()	5✔
114	assert self._started, (	5✔
115	"cannot assert that pantsd is running. Try calling assert_started before calling this method."
116	)
117	assert self.is_alive(), "pantsd was not alive."	5✔
118	return self.pid	5✔
119
120	def current_memory_usage(self):	5✔
121	"""Return the current memory usage of the pantsd process (which must be running)
122
123	:return: memory usage in bytes
124	"""
125	self.assert_running()	×
126	return self._as_process().memory_info()[0]	×
127
128	def assert_running(self):	5✔
129	if not self._started:	×
130	return self.assert_started()	×
131	else:
132	return self._check_pantsd_is_alive()	×
133
134	def assert_stopped(self):	5✔
135	self._log()	5✔
136	assert self._started, (	5✔
137	"cannot assert pantsd stoppage. Try calling assert_started before calling this method."
138	)
139	for _ in attempts("pantsd should be stopped!"):	5✔
140	if self.is_dead():	5✔
141	break	5✔
142
143
144	@dataclass(frozen=True)	5✔
145	class PantsdRunContext:	5✔
146	runner: Callable[..., Any]	5✔
147	checker: PantsDaemonMonitor	5✔
148	workdir: str	5✔
149	pantsd_config: dict[str, Any]	5✔
150
151
152	class PantsDaemonIntegrationTestBase(unittest.TestCase):	5✔
153	@staticmethod	5✔
154	def run_pants(args, *kwargs) -> PantsResult:	5✔
155	# We set our own ad-hoc pantsd configuration in most of these tests.
156	return run_pants(args, {kwargs, *{"use_pantsd": False}})	×
157
158	@staticmethod	5✔
159	def run_pants_with_workdir(args, *kwargs) -> PantsResult:	5✔
160	# We set our own ad-hoc pantsd configuration in most of these tests.
161	return run_pants_with_workdir(args, {kwargs, *{"use_pantsd": False}})	5✔
162
163	@staticmethod	5✔
164	def run_pants_with_workdir_without_waiting(args, *kwargs) -> PantsJoinHandle:	5✔
165	# We set our own ad-hoc pantsd configuration in most of these tests.
166	return run_pants_with_workdir_without_waiting(args, {kwargs, *{"use_pantsd": False}})	1✔
167
168	@contextmanager	5✔
169	def pantsd_test_context(	5✔
170	self, *, log_level: str = "info", extra_config: dict[str, Any] \| None = None
171	) -> Iterator[tuple[str, dict[str, Any], PantsDaemonMonitor]]:
172	with temporary_dir(root_dir=os.getcwd()) as dot_pants_dot_d:	5✔
173	pid_dir = os.path.join(dot_pants_dot_d, "pids")	5✔
174	workdir = os.path.join(dot_pants_dot_d, "workdir")	5✔
175	print(f"\npantsd log is {workdir}/pants.log")	5✔
176	pantsd_config = {	5✔
177	"GLOBAL": {
178	"pantsd": True,
179	"level": log_level,
180	"pants_subprocessdir": pid_dir,
181	"backend_packages": [
182	# Provide goals used by various tests.
183	"pants.backend.python",
184	"pants.backend.python.lint.flake8",
185	],
186	},
187	"python": {
188	"interpreter_constraints": "['>=3.14,<3.15']",
189	},
190	}
191
192	if extra_config:	5✔
193	recursively_update(pantsd_config, extra_config)	×
194	print(f">>> config: \n{pantsd_config}\n")	5✔
195
196	checker = PantsDaemonMonitor(pid_dir)	5✔
197	kill_daemon(pid_dir)	5✔
198	try:	5✔
199	yield workdir, pantsd_config, checker	5✔
200	kill_daemon(pid_dir)	5✔
201	checker.assert_stopped()	5✔
202	finally:
203	banner("BEGIN pants.log")	5✔
204	for line in read_pants_log(workdir):	5✔
205	print(line)	5✔
206	banner("END pants.log")	5✔
207
208	@contextmanager	5✔
209	def pantsd_successful_run_context(self, args, *kwargs) -> Iterator[PantsdRunContext]:	5✔
210	with self.pantsd_run_context(args, success=True, *kwargs) as context: # type: ignore[misc]	5✔
211	yield context	5✔
212
213	@contextmanager	5✔
214	def pantsd_run_context(	5✔
215	self,
216	log_level: str = "info",
217	extra_config: dict[str, Any] \| None = None,
218	extra_env: dict[str, str] \| None = None,
219	success: bool = True,
220	) -> Iterator[PantsdRunContext]:
221	with self.pantsd_test_context(log_level=log_level, extra_config=extra_config) as (	5✔
222	workdir,
223	pantsd_config,
224	checker,
225	):
226	runner = functools.partial(	5✔
227	self.assert_runner,
228	workdir,
229	pantsd_config,
230	extra_env=extra_env,
231	success=success,
232	)
233	yield PantsdRunContext(	5✔
234	runner=runner, checker=checker, workdir=workdir, pantsd_config=pantsd_config
235	)
236
237	def _run_count(self, workdir):	5✔
238	run_tracker_dir = os.path.join(workdir, "run-tracker")	5✔
239	if os.path.isdir(run_tracker_dir):	5✔
240	return len([f for f in os.listdir(run_tracker_dir) if f != "latest"])	5✔
241	else:
242	return 0	5✔
243
244	def assert_runner(	5✔
245	self,
246	workdir: str,
247	config,
248	cmd,
249	extra_config=None,
250	extra_env=None,
251	success=True,
252	expected_runs: int = 1,
253	):
254	combined_config = config.copy()	5✔
255	recursively_update(combined_config, extra_config or {})	5✔
256	print(	5✔
257	bold(
258	cyan(
259	"\nrunning: ./pants {} (config={}) (extra_env={})".format(
260	" ".join(cmd), combined_config, extra_env
261	)
262	)
263	)
264	)
265	run_count = self._run_count(workdir)	5✔
266	start_time = time.time()	5✔
267	run = self.run_pants_with_workdir(	5✔
268	cmd, workdir=workdir, config=combined_config, extra_env=extra_env or {}
269	)
270	elapsed = time.time() - start_time	5✔
271	print(bold(cyan(f"\ncompleted in {elapsed} seconds")))	5✔
272
273	if success:	5✔
274	run.assert_success()	5✔
275	else:
276	run.assert_failure()	×
277
278	runs_created = self._run_count(workdir) - run_count	5✔
279	self.assertEqual(	5✔
280	runs_created,
281	expected_runs,
282	f"Expected {expected_runs} RunTracker run(s) to be created per pantsd run: was {runs_created}",
283	)
284
285	return run	5✔

pantsbuild / pants / 26342152999

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous