22360764254

Committed 24 Feb 2026 04:46PM UTC coverage: 88.798% (-4.1%) from 92.935%

Build # 22360764254

Build Type

Pull #23133

github

Committed by

web-flow

Commit Message

Merge 4c056364c into 4d038bd74

Pull Request Pull Request #23133: Add buildctl engine

Coverage Stats

181 of 264 new or added lines in 8 files covered. (68.56%)

3184 existing lines in 145 files now uncovered.

77555 of 87339 relevant lines covered (88.8%)

3.34 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

56.63

/src/python/pants/backend/docker/util_rules/binaries.py

# Copyright 2026 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
import os
from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import TypeVar, cast

from pants.backend.docker.subsystems.docker_options import DockerOptions
from pants.backend.docker.util_rules.docker_build_args import DockerBuildArgs
from pants.core.util_rules.system_binaries import (
    BinaryPath,
    BinaryPathRequest,
    BinaryPathTest,
    BinaryShims,
    BinaryShimsRequest,
    create_binary_shims,
    find_binary,
)
from pants.engine.fs import Digest
from pants.engine.process import Process, ProcessCacheScope
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
from pants.util.logging import LogLevel
from pants.util.strutil import pluralize

T = TypeVar("T", bound="BaseBinary")


@dataclass(frozen=True)
class BaseBinary(BinaryPath, ABC):
    """Base class for all binary paths."""

    global_options: tuple[str, ...]
    extra_env: Mapping[str, str]
    extra_input_digests: Mapping[str, Digest] | None

    def __init__(
        self,
        path: str,
        fingerprint: str | None = None,
        global_options: tuple[str, ...] = (),
        extra_env: Mapping[str, str] | None = None,
        extra_input_digests: Mapping[str, Digest] | None = None,
    ) -> None:
        object.__setattr__(self, "global_options", global_options)
        object.__setattr__(self, "extra_env", {} if extra_env is None else extra_env)
        object.__setattr__(self, "extra_input_digests", extra_input_digests)
        super().__init__(path, fingerprint)

    def _get_process_environment(self, env: Mapping[str, str]) -> Mapping[str, str]:
        if not self.extra_env:
            return env

        res = {**self.extra_env, **env}

        # Merge the PATH entries, in case they are present in both `env` and `self.extra_env`.
        res["PATH"] = os.pathsep.join(
            p for p in (m.get("PATH") for m in (self.extra_env, env)) if p
        )
        return res

    @abstractmethod
    def build_image(
        self,
        tags: tuple[str, ...],
        digest: Digest,
        dockerfile: str,
        build_args: DockerBuildArgs,
        context_root: str,
        env: Mapping[str, str],
        extra_args: tuple[str, ...] = (),
    ) -> Process: ...


class _DockerPodmanMixin(BaseBinary):
    def build_image(
        self,
        tags: tuple[str, ...],
        digest: Digest,
        dockerfile: str,
        build_args: DockerBuildArgs,
        context_root: str,
        env: Mapping[str, str],
        extra_args: tuple[str, ...] = (),
    ) -> Process:
        args = [self.path, *self.global_options, "build", *extra_args]

        for tag in tags:
            args.extend(["--tag", tag])

        for build_arg in build_args:
            args.extend(["--build-arg", build_arg])

        args.extend(["--file", dockerfile])

        # Docker context root.
        args.append(context_root)

        return Process(
            argv=tuple(args),
            description=(
                f"Building docker image {tags[0]}"
                + (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
            ),
            env=self._get_process_environment(env),
            input_digest=digest,
            immutable_input_digests=self.extra_input_digests,
            # We must run the docker build commands every time, even if nothing has changed,
            # in case the user ran `docker image rm` outside of Pants.
            cache_scope=ProcessCacheScope.PER_SESSION,
        )

    def push_image(self, tag: str, env: Mapping[str, str] | None = None) -> Process:
        return Process(
            argv=(self.path, *self.global_options, "push", tag),
            cache_scope=ProcessCacheScope.PER_SESSION,
            description=f"Pushing docker image {tag}",
            env=self._get_process_environment(env or {}),
            immutable_input_digests=self.extra_input_digests,
        )

    def run_image(
        self,
        tag: str,
        *,
        docker_run_args: tuple[str, ...] | None = None,
        image_args: tuple[str, ...] | None = None,
        env: Mapping[str, str] | None = None,
    ) -> Process:
        return Process(
            argv=(
                self.path,
                *self.global_options,
                "run",
                *(docker_run_args or []),
                tag,
                *(image_args or []),
            ),
            cache_scope=ProcessCacheScope.PER_SESSION,
            description=f"Running docker image {tag}",
            env=self._get_process_environment(env or {}),
            immutable_input_digests=self.extra_input_digests,
        )


class DockerBinary(_DockerPodmanMixin):
    """The `docker` binary."""


class PodmanBinary(_DockerPodmanMixin):
    """The `podman` binary."""


class BuildctlBinary(BaseBinary):
    """The `buildctl` binary."""

    def build_image(
        self,
        tags: tuple[str, ...],
        digest: Digest,
        dockerfile: str,
        build_args: DockerBuildArgs,
        context_root: str,
        env: Mapping[str, str],
        extra_args: tuple[str, ...] = (),
    ) -> Process:
        args = [
            self.path,
            *self.global_options,
            "build",
            "--frontend",
            "dockerfile.v0",
            "--local",
            f"context={context_root}",
            "--local",
            f"dockerfile={os.path.dirname(dockerfile)}",
            "--opt",
            f"filename={os.path.basename(dockerfile)}",
            *extra_args,
        ]

        for build_arg in build_args:
            args.extend(["--opt", f"build-arg:{build_arg}"])

        for tag in tags:
            args.extend(["--output", f"type=image,name={tag},push=true"])

        return Process(
            argv=tuple(args),
            description=(
                f"Building docker image {tags[0]}"
                + (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
            ),
            env=self._get_process_environment(env),
            input_digest=digest,
            immutable_input_digests=self.extra_input_digests,
            cache_scope=ProcessCacheScope.PER_SESSION,
        )


async def _get_docker_tools_shims(
    *,
    tools: Sequence[str],
    optional_tools: Sequence[str],
    search_path: Sequence[str],
    rationale: str,
) -> BinaryShims:
    all_binary_first_paths: list[BinaryPath] = []

    if tools:
        tools_requests = [
            BinaryPathRequest(binary_name=binary_name, search_path=search_path)
            for binary_name in tools
        ]

        tools_paths = await concurrently(
            find_binary(tools_request, **implicitly()) for tools_request in tools_requests
        )

        all_binary_first_paths.extend(
            [
                path.first_path_or_raise(request, rationale=rationale)
                for request, path in zip(tools_requests, tools_paths)
            ]
        )

    if optional_tools:
        optional_tools_requests = [
            BinaryPathRequest(binary_name=binary_name, search_path=search_path)
            for binary_name in optional_tools
        ]

        optional_tools_paths = await concurrently(
            find_binary(optional_tools_request, **implicitly())
            for optional_tools_request in optional_tools_requests
        )

        all_binary_first_paths.extend(
            [
                cast(BinaryPath, path.first_path)  # safe since we check for non-empty paths below
                for path in optional_tools_paths
                if path.paths
            ]
        )

    tools_shims = await create_binary_shims(
        BinaryShimsRequest.for_paths(
            *all_binary_first_paths,
            rationale=rationale,
        ),
        **implicitly(),
    )

    return tools_shims


async def get_binary(
    binary_name: str,
    binary_cls: type[T],
    docker_options: DockerOptions,
    docker_options_env_aware: DockerOptions.EnvironmentAware,
) -> T:
    search_path = docker_options_env_aware.executable_search_path

    request = BinaryPathRequest(
        binary_name=binary_name,
        search_path=search_path,
        test=BinaryPathTest(args=["-v"]),
    )
    paths = await find_binary(request, **implicitly())
    first_path = paths.first_path_or_raise(request, rationale="interact with the docker daemon")

    if not docker_options.tools and not docker_options.optional_tools:
        return binary_cls(
            first_path.path, first_path.fingerprint, global_options=docker_options.global_options
        )

    tools_shims = await _get_docker_tools_shims(
        tools=docker_options.tools,
        optional_tools=docker_options.optional_tools,
        search_path=search_path,
        rationale=f"use {binary_name}",
    )
    return binary_cls(
        first_path.path,
        first_path.fingerprint,
        global_options=docker_options.global_options,
        extra_env={"PATH": tools_shims.path_component},
        extra_input_digests=tools_shims.immutable_input_digests,
    )


@rule(desc="Finding the `docker` binary and related tooling", level=LogLevel.DEBUG)
async def get_docker(
    docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
) -> DockerBinary:
    return await get_binary("docker", DockerBinary, docker_options, docker_options_env_aware)


@rule(desc="Finding the `podman` binary and related tooling", level=LogLevel.DEBUG)
async def get_podman(
    docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
) -> PodmanBinary:
    return await get_binary("podman", PodmanBinary, docker_options, docker_options_env_aware)


@rule(desc="Finding the `buildctl` binary and related tooling", level=LogLevel.DEBUG)
async def get_buildctl(
    docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
) -> BuildctlBinary:
    return await get_binary("buildctl", BuildctlBinary, docker_options, docker_options_env_aware)


def rules():
    return collect_rules()

1	# Copyright 2026 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	import os	4✔
4	from abc import ABC, abstractmethod	4✔
5	from collections.abc import Mapping, Sequence	4✔
6	from dataclasses import dataclass	4✔
7	from typing import TypeVar, cast	4✔
8
9	from pants.backend.docker.subsystems.docker_options import DockerOptions	4✔
10	from pants.backend.docker.util_rules.docker_build_args import DockerBuildArgs	4✔
11	from pants.core.util_rules.system_binaries import (	4✔
12	BinaryPath,
13	BinaryPathRequest,
14	BinaryPathTest,
15	BinaryShims,
16	BinaryShimsRequest,
17	create_binary_shims,
18	find_binary,
19	)
20	from pants.engine.fs import Digest	4✔
21	from pants.engine.process import Process, ProcessCacheScope	4✔
22	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	4✔
23	from pants.util.logging import LogLevel	4✔
24	from pants.util.strutil import pluralize	4✔
25
26	T = TypeVar("T", bound="BaseBinary")	4✔
27
28
29	@dataclass(frozen=True)	4✔
30	class BaseBinary(BinaryPath, ABC):	4✔
31	"""Base class for all binary paths."""
32
33	global_options: tuple[str, ...]
34	extra_env: Mapping[str, str]
35	extra_input_digests: Mapping[str, Digest] \| None
36
37	def __init__(	4✔
38	self,
39	path: str,
40	fingerprint: str \| None = None,
41	global_options: tuple[str, ...] = (),
42	extra_env: Mapping[str, str] \| None = None,
43	extra_input_digests: Mapping[str, Digest] \| None = None,
44	) -> None:
45	object.__setattr__(self, "global_options", global_options)	2✔
46	object.__setattr__(self, "extra_env", {} if extra_env is None else extra_env)	2✔
47	object.__setattr__(self, "extra_input_digests", extra_input_digests)	2✔
48	super().__init__(path, fingerprint)	2✔
49
50	def _get_process_environment(self, env: Mapping[str, str]) -> Mapping[str, str]:	4✔
51	if not self.extra_env:	×
52	return env	×
53
54	res = {self.extra_env, env}	×
55
56	# Merge the PATH entries, in case they are present in both `env` and `self.extra_env`.
57	res["PATH"] = os.pathsep.join(	×
58	p for p in (m.get("PATH") for m in (self.extra_env, env)) if p
59	)
NEW 60	return res	×
61
62	@abstractmethod
63	def build_image(
64	self,
65	tags: tuple[str, ...],
66	digest: Digest,
67	dockerfile: str,
68	build_args: DockerBuildArgs,
69	context_root: str,
70	env: Mapping[str, str],
71	extra_args: tuple[str, ...] = (),
72	) -> Process: ...
73
74
75	class _DockerPodmanMixin(BaseBinary):	4✔
76	def build_image(	4✔
77	self,
78	tags: tuple[str, ...],
79	digest: Digest,
80	dockerfile: str,
81	build_args: DockerBuildArgs,
82	context_root: str,
83	env: Mapping[str, str],
84	extra_args: tuple[str, ...] = (),
85	) -> Process:
NEW 86	args = [self.path, self.global_options, "build", extra_args]	×
87
NEW 88	for tag in tags:	×
89	args.extend(["--tag", tag])	×
90
91	for build_arg in build_args:	×
92	args.extend(["--build-arg", build_arg])	×
93
94	args.extend(["--file", dockerfile])	×
95
96	# Docker context root.
97	args.append(context_root)	×
98
99	return Process(	×
100	argv=tuple(args),
101	description=(
102	f"Building docker image {tags[0]}"
103	+ (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
104	),
105	env=self._get_process_environment(env),
106	input_digest=digest,
107	immutable_input_digests=self.extra_input_digests,
108	# We must run the docker build commands every time, even if nothing has changed,
109	# in case the user ran `docker image rm` outside of Pants.
110	cache_scope=ProcessCacheScope.PER_SESSION,
111	)
112
113	def push_image(self, tag: str, env: Mapping[str, str] \| None = None) -> Process:	4✔
114	return Process(	×
115	argv=(self.path, *self.global_options, "push", tag),
116	cache_scope=ProcessCacheScope.PER_SESSION,
117	description=f"Pushing docker image {tag}",
118	env=self._get_process_environment(env or {}),
119	immutable_input_digests=self.extra_input_digests,
120	)
121
122	def run_image(	4✔
123	self,
124	tag: str,
125	*,
126	docker_run_args: tuple[str, ...] \| None = None,
127	image_args: tuple[str, ...] \| None = None,
128	env: Mapping[str, str] \| None = None,
129	) -> Process:
130	return Process(	×
131	argv=(
132	self.path,
133	*self.global_options,
134	"run",
135	*(docker_run_args or []),
136	tag,
137	*(image_args or []),
138	),
139	cache_scope=ProcessCacheScope.PER_SESSION,
140	description=f"Running docker image {tag}",
141	env=self._get_process_environment(env or {}),
142	immutable_input_digests=self.extra_input_digests,
143	)
144
145
146	class DockerBinary(_DockerPodmanMixin):	4✔
147	"""The `docker` binary."""
148
149
150	class PodmanBinary(_DockerPodmanMixin):	4✔
151	"""The `podman` binary."""
152
153
154	class BuildctlBinary(BaseBinary):	4✔
155	"""The `buildctl` binary."""
156
157	def build_image(	4✔
158	self,
159	tags: tuple[str, ...],
160	digest: Digest,
161	dockerfile: str,
162	build_args: DockerBuildArgs,
163	context_root: str,
164	env: Mapping[str, str],
165	extra_args: tuple[str, ...] = (),
166	) -> Process:
NEW 167	args = [	×
168	self.path,
169	*self.global_options,
170	"build",
171	"--frontend",
172	"dockerfile.v0",
173	"--local",
174	f"context={context_root}",
175	"--local",
176	f"dockerfile={os.path.dirname(dockerfile)}",
177	"--opt",
178	f"filename={os.path.basename(dockerfile)}",
179	*extra_args,
180	]
181
NEW 182	for build_arg in build_args:	×
NEW 183	args.extend(["--opt", f"build-arg:{build_arg}"])	×
184
NEW 185	for tag in tags:	×
NEW 186	args.extend(["--output", f"type=image,name={tag},push=true"])	×
187
NEW 188	return Process(	×
189	argv=tuple(args),
190	description=(
191	f"Building docker image {tags[0]}"
192	+ (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
193	),
194	env=self._get_process_environment(env),
195	input_digest=digest,
196	immutable_input_digests=self.extra_input_digests,
197	cache_scope=ProcessCacheScope.PER_SESSION,
198	)
199
200
201	async def _get_docker_tools_shims(	4✔
202	*,
203	tools: Sequence[str],
204	optional_tools: Sequence[str],
205	search_path: Sequence[str],
206	rationale: str,
207	) -> BinaryShims:
208	all_binary_first_paths: list[BinaryPath] = []	×
209
210	if tools:	×
211	tools_requests = [	×
212	BinaryPathRequest(binary_name=binary_name, search_path=search_path)
213	for binary_name in tools
214	]
215
216	tools_paths = await concurrently(	×
217	find_binary(tools_request, **implicitly()) for tools_request in tools_requests
218	)
219
220	all_binary_first_paths.extend(	×
221	[
222	path.first_path_or_raise(request, rationale=rationale)
223	for request, path in zip(tools_requests, tools_paths)
224	]
225	)
226
227	if optional_tools:	×
228	optional_tools_requests = [	×
229	BinaryPathRequest(binary_name=binary_name, search_path=search_path)
230	for binary_name in optional_tools
231	]
232
233	optional_tools_paths = await concurrently(	×
234	find_binary(optional_tools_request, **implicitly())
235	for optional_tools_request in optional_tools_requests
236	)
237
238	all_binary_first_paths.extend(	×
239	[
240	cast(BinaryPath, path.first_path) # safe since we check for non-empty paths below
241	for path in optional_tools_paths
242	if path.paths
243	]
244	)
245
246	tools_shims = await create_binary_shims(	×
247	BinaryShimsRequest.for_paths(
248	*all_binary_first_paths,
249	rationale=rationale,
250	),
251	**implicitly(),
252	)
253
254	return tools_shims	×
255
256
257	async def get_binary(	4✔
258	binary_name: str,
259	binary_cls: type[T],
260	docker_options: DockerOptions,
261	docker_options_env_aware: DockerOptions.EnvironmentAware,
262	) -> T:
263	search_path = docker_options_env_aware.executable_search_path	2✔
264
265	request = BinaryPathRequest(	2✔
266	binary_name=binary_name,
267	search_path=search_path,
268	test=BinaryPathTest(args=["-v"]),
269	)
270	paths = await find_binary(request, **implicitly())	2✔
271	first_path = paths.first_path_or_raise(request, rationale="interact with the docker daemon")	2✔
272
273	if not docker_options.tools and not docker_options.optional_tools:	2✔
274	return binary_cls(	2✔
275	first_path.path, first_path.fingerprint, global_options=docker_options.global_options
276	)
277
278	tools_shims = await _get_docker_tools_shims(	×
279	tools=docker_options.tools,
280	optional_tools=docker_options.optional_tools,
281	search_path=search_path,
282	rationale=f"use {binary_name}",
283	)
284	return binary_cls(	×
285	first_path.path,
286	first_path.fingerprint,
287	global_options=docker_options.global_options,
288	extra_env={"PATH": tools_shims.path_component},
289	extra_input_digests=tools_shims.immutable_input_digests,
290	)
291
292
293	@rule(desc="Finding the `docker` binary and related tooling", level=LogLevel.DEBUG)	4✔
294	async def get_docker(	4✔
295	docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
296	) -> DockerBinary:
297	return await get_binary("docker", DockerBinary, docker_options, docker_options_env_aware)	2✔
298
299
300	@rule(desc="Finding the `podman` binary and related tooling", level=LogLevel.DEBUG)	4✔
301	async def get_podman(	4✔
302	docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
303	) -> PodmanBinary:
NEW 304	return await get_binary("podman", PodmanBinary, docker_options, docker_options_env_aware)	×
305
306
307	@rule(desc="Finding the `buildctl` binary and related tooling", level=LogLevel.DEBUG)	4✔
308	async def get_buildctl(	4✔
309	docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
310	) -> BuildctlBinary:
311	return await get_binary("buildctl", BuildctlBinary, docker_options, docker_options_env_aware)	×
312
313
314	def rules():	4✔
315	return collect_rules()	4✔

pantsbuild / pants / 22360764254

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous