23505168284

Committed 24 Mar 2026 06:11PM UTC coverage: 92.618% (-0.3%) from 92.918%

Build # 23505168284

Build Type

Pull #23133

github

Committed by

web-flow

Commit Message

Merge e8047d851 into b84b29b9b

Pull Request Pull Request #23133: Add buildctl engine

Coverage Stats

215 of 296 new or added lines in 13 files covered. (72.64%)

233 existing lines in 12 files now uncovered.

91363 of 98645 relevant lines covered (92.62%)

4.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.9

/src/python/pants/backend/docker/util_rules/binaries.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
import os
from abc import ABC
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Protocol, TypeVar, cast

from pants.backend.docker.subsystems.docker_options import DockerOptions
from pants.backend.docker.util_rules.docker_build_args import DockerBuildArgs
from pants.core.util_rules.system_binaries import (
    BinaryPath,
    BinaryPathRequest,
    BinaryPathTest,
    BinaryShims,
    BinaryShimsRequest,
    create_binary_shims,
    find_binary,
)
from pants.engine.fs import Digest
from pants.engine.process import Process, ProcessCacheScope
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
from pants.util.logging import LogLevel
from pants.util.strutil import pluralize

T = TypeVar("T", bound="BaseBinary")


@dataclass(frozen=True)
class BaseBinary(BinaryPath, ABC):
    """Base class for all binary paths."""

    extra_env: Mapping[str, str]
    extra_input_digests: Mapping[str, Digest] | None

    def __init__(
        self,
        path: str,
        fingerprint: str | None = None,
        extra_env: Mapping[str, str] | None = None,
        extra_input_digests: Mapping[str, Digest] | None = None,
    ) -> None:
        object.__setattr__(self, "extra_env", {} if extra_env is None else extra_env)
        object.__setattr__(self, "extra_input_digests", extra_input_digests)
        super().__init__(path, fingerprint)

    def _get_process_environment(self, env: Mapping[str, str]) -> Mapping[str, str]:
        if not self.extra_env:
            return env

        res = {**self.extra_env, **env}

        # Merge the PATH entries, in case they are present in both `env` and `self.extra_env`.
        res["PATH"] = os.pathsep.join(
            p for p in (m.get("PATH") for m in (self.extra_env, env)) if p
        )
        return res


class BuildBinaryProtocol(Protocol):
    def build_image(
        self,
        tags: tuple[str, ...],
        digest: Digest,
        dockerfile: str,
        build_args: DockerBuildArgs,
        context_root: str,
        env: Mapping[str, str],
        extra_args: tuple[str, ...] = (),
    ) -> Process: ...


class PushBinaryProtocol(Protocol):
    def push_image(self, tag: str, env: Mapping[str, str] | None = None) -> Process: ...


class _DockerPodmanMixin(BaseBinary):
    def build_image(
        self,
        tags: tuple[str, ...],
        digest: Digest,
        dockerfile: str,
        build_args: DockerBuildArgs,
        context_root: str,
        env: Mapping[str, str],
        extra_args: tuple[str, ...] = (),
    ) -> Process:
        args = [self.path, "build", *extra_args]

        for tag in tags:
            args.extend(["--tag", tag])

        for build_arg in build_args:
            args.extend(["--build-arg", build_arg])

        args.extend(["--file", dockerfile])

        # Docker context root.
        args.append(context_root)

        return Process(
            argv=tuple(args),
            description=(
                f"Building docker image {tags[0]}"
                + (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
            ),
            env=self._get_process_environment(env),
            input_digest=digest,
            immutable_input_digests=self.extra_input_digests,
            # We must run the docker build commands every time, even if nothing has changed,
            # in case the user ran `docker image rm` outside of Pants.
            cache_scope=ProcessCacheScope.PER_SESSION,
        )

    def push_image(self, tag: str, env: Mapping[str, str] | None = None) -> Process:
        return Process(
            argv=(self.path, "push", tag),
            cache_scope=ProcessCacheScope.PER_SESSION,
            description=f"Pushing docker image {tag}",
            env=self._get_process_environment(env or {}),
            immutable_input_digests=self.extra_input_digests,
        )

    def run_image(
        self,
        tag: str,
        *,
        docker_run_args: tuple[str, ...] | None = None,
        image_args: tuple[str, ...] | None = None,
        env: Mapping[str, str] | None = None,
    ) -> Process:
        return Process(
            argv=(
                self.path,
                "run",
                *(docker_run_args or []),
                tag,
                *(image_args or []),
            ),
            cache_scope=ProcessCacheScope.PER_SESSION,
            description=f"Running docker image {tag}",
            env=self._get_process_environment(env or {}),
            immutable_input_digests=self.extra_input_digests,
        )


class DockerBinary(_DockerPodmanMixin):
    """The `docker` binary."""


class PodmanBinary(_DockerPodmanMixin):
    """The `podman` binary."""


class BuildctlBinary(BaseBinary):
    """The `buildctl` binary."""

    def build_image(
        self,
        tags: tuple[str, ...],
        digest: Digest,
        dockerfile: str,
        build_args: DockerBuildArgs,
        context_root: str,
        env: Mapping[str, str],
        extra_args: tuple[str, ...] = (),
    ) -> Process:
        args = [
            self.path,
            "build",
            "--frontend",
            "dockerfile.v0",
            "--local",
            f"context={context_root}",
            "--local",
            f"dockerfile={os.path.dirname(dockerfile)}",
            "--opt",
            f"filename={os.path.basename(dockerfile)}",
            *extra_args,
        ]

        for build_arg in build_args:
            args.extend(["--opt", f"build-arg:{build_arg}"])

        for tag in tags:
            args.extend(["--output", f"type=image,name={tag},push=true"])

        return Process(
            argv=tuple(args),
            description=(
                f"Building docker image {tags[0]}"
                + (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
            ),
            env=self._get_process_environment(env),
            input_digest=digest,
            immutable_input_digests=self.extra_input_digests,
            cache_scope=ProcessCacheScope.PER_SESSION,
        )


async def _get_docker_tools_shims(
    *,
    tools: Sequence[str],
    optional_tools: Sequence[str],
    search_path: Sequence[str],
    rationale: str,
) -> BinaryShims:
    all_binary_first_paths: list[BinaryPath] = []

    if tools:
        tools_requests = [
            BinaryPathRequest(binary_name=binary_name, search_path=search_path)
            for binary_name in tools
        ]

        tools_paths = await concurrently(
            find_binary(tools_request, **implicitly()) for tools_request in tools_requests
        )

        all_binary_first_paths.extend(
            [
                path.first_path_or_raise(request, rationale=rationale)
                for request, path in zip(tools_requests, tools_paths)
            ]
        )

    if optional_tools:
        optional_tools_requests = [
            BinaryPathRequest(binary_name=binary_name, search_path=search_path)
            for binary_name in optional_tools
        ]

        optional_tools_paths = await concurrently(
            find_binary(optional_tools_request, **implicitly())
            for optional_tools_request in optional_tools_requests
        )

        all_binary_first_paths.extend(
            [
                cast(BinaryPath, path.first_path)  # safe since we check for non-empty paths below
                for path in optional_tools_paths
                if path.paths
            ]
        )

    tools_shims = await create_binary_shims(
        BinaryShimsRequest.for_paths(
            *all_binary_first_paths,
            rationale=rationale,
        ),
        **implicitly(),
    )

    return tools_shims


async def get_binary(
    binary_name: str,
    binary_cls: type[T],
    docker_options: DockerOptions,
    docker_options_env_aware: DockerOptions.EnvironmentAware,
) -> T:
    search_path = docker_options_env_aware.executable_search_path

    request = BinaryPathRequest(
        binary_name=binary_name,
        search_path=search_path,
        test=BinaryPathTest(args=["-v"]),
    )
    paths = await find_binary(request, **implicitly())
    first_path = paths.first_path_or_raise(request, rationale="interact with the docker daemon")

    if not docker_options.tools and not docker_options.optional_tools:
        return binary_cls(first_path.path, first_path.fingerprint)

    tools_shims = await _get_docker_tools_shims(
        tools=docker_options.tools,
        optional_tools=docker_options.optional_tools,
        search_path=search_path,
        rationale=f"use {binary_name}",
    )
    return binary_cls(
        first_path.path,
        first_path.fingerprint,
        extra_env={"PATH": tools_shims.path_component},
        extra_input_digests=tools_shims.immutable_input_digests,
    )


@rule(desc="Finding the `docker` binary and related tooling", level=LogLevel.DEBUG)
async def get_docker(
    docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
) -> DockerBinary:
    return await get_binary("docker", DockerBinary, docker_options, docker_options_env_aware)


@rule(desc="Finding the `podman` binary and related tooling", level=LogLevel.DEBUG)
async def get_podman(
    docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
) -> PodmanBinary:
    return await get_binary("podman", PodmanBinary, docker_options, docker_options_env_aware)


@rule(desc="Finding the `buildctl` binary and related tooling", level=LogLevel.DEBUG)
async def get_buildctl(
    docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
) -> BuildctlBinary:
    return await get_binary("buildctl", BuildctlBinary, docker_options, docker_options_env_aware)


def rules():
    return collect_rules()

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	import os	9✔
4	from abc import ABC	9✔
5	from collections.abc import Mapping, Sequence	9✔
6	from dataclasses import dataclass	9✔
7	from typing import Protocol, TypeVar, cast	9✔
8
9	from pants.backend.docker.subsystems.docker_options import DockerOptions	9✔
10	from pants.backend.docker.util_rules.docker_build_args import DockerBuildArgs	9✔
11	from pants.core.util_rules.system_binaries import (	9✔
12	BinaryPath,
13	BinaryPathRequest,
14	BinaryPathTest,
15	BinaryShims,
16	BinaryShimsRequest,
17	create_binary_shims,
18	find_binary,
19	)
20	from pants.engine.fs import Digest	9✔
21	from pants.engine.process import Process, ProcessCacheScope	9✔
22	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	9✔
23	from pants.util.logging import LogLevel	9✔
24	from pants.util.strutil import pluralize	9✔
25
26	T = TypeVar("T", bound="BaseBinary")	9✔
27
28
29	@dataclass(frozen=True)	9✔
30	class BaseBinary(BinaryPath, ABC):	9✔
31	"""Base class for all binary paths."""
32
33	extra_env: Mapping[str, str]
34	extra_input_digests: Mapping[str, Digest] \| None
35
36	def __init__(	9✔
37	self,
38	path: str,
39	fingerprint: str \| None = None,
40	extra_env: Mapping[str, str] \| None = None,
41	extra_input_digests: Mapping[str, Digest] \| None = None,
42	) -> None:
43	object.__setattr__(self, "extra_env", {} if extra_env is None else extra_env)	5✔
44	object.__setattr__(self, "extra_input_digests", extra_input_digests)	5✔
45	super().__init__(path, fingerprint)	5✔
46
47	def _get_process_environment(self, env: Mapping[str, str]) -> Mapping[str, str]:	9✔
48	if not self.extra_env:	1✔
49	return env	1✔
50
51	res = {self.extra_env, env}	×
52
53	# Merge the PATH entries, in case they are present in both `env` and `self.extra_env`.
54	res["PATH"] = os.pathsep.join(	×
55	p for p in (m.get("PATH") for m in (self.extra_env, env)) if p
56	)
57	return res	×
58
59
60	class BuildBinaryProtocol(Protocol):	9✔
61	def build_image(
62	self,
63	tags: tuple[str, ...],
64	digest: Digest,
65	dockerfile: str,
66	build_args: DockerBuildArgs,
67	context_root: str,
68	env: Mapping[str, str],
69	extra_args: tuple[str, ...] = (),
70	) -> Process: ...
71
72
73	class PushBinaryProtocol(Protocol):	9✔
74	def push_image(self, tag: str, env: Mapping[str, str] \| None = None) -> Process: ...
75
76
77	class _DockerPodmanMixin(BaseBinary):	9✔
78	def build_image(	9✔
79	self,
80	tags: tuple[str, ...],
81	digest: Digest,
82	dockerfile: str,
83	build_args: DockerBuildArgs,
84	context_root: str,
85	env: Mapping[str, str],
86	extra_args: tuple[str, ...] = (),
87	) -> Process:
88	args = [self.path, "build", *extra_args]	1✔
89
90	for tag in tags:	1✔
91	args.extend(["--tag", tag])	1✔
92
93	for build_arg in build_args:	1✔
94	args.extend(["--build-arg", build_arg])	1✔
95
96	args.extend(["--file", dockerfile])	1✔
97
98	# Docker context root.
99	args.append(context_root)	1✔
100
101	return Process(	1✔
102	argv=tuple(args),
103	description=(
104	f"Building docker image {tags[0]}"
105	+ (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
106	),
107	env=self._get_process_environment(env),
108	input_digest=digest,
109	immutable_input_digests=self.extra_input_digests,
110	# We must run the docker build commands every time, even if nothing has changed,
111	# in case the user ran `docker image rm` outside of Pants.
112	cache_scope=ProcessCacheScope.PER_SESSION,
113	)
114
115	def push_image(self, tag: str, env: Mapping[str, str] \| None = None) -> Process:	9✔
116	return Process(	1✔
117	argv=(self.path, "push", tag),
118	cache_scope=ProcessCacheScope.PER_SESSION,
119	description=f"Pushing docker image {tag}",
120	env=self._get_process_environment(env or {}),
121	immutable_input_digests=self.extra_input_digests,
122	)
123
124	def run_image(	9✔
125	self,
126	tag: str,
127	*,
128	docker_run_args: tuple[str, ...] \| None = None,
129	image_args: tuple[str, ...] \| None = None,
130	env: Mapping[str, str] \| None = None,
131	) -> Process:
132	return Process(	1✔
133	argv=(
134	self.path,
135	"run",
136	*(docker_run_args or []),
137	tag,
138	*(image_args or []),
139	),
140	cache_scope=ProcessCacheScope.PER_SESSION,
141	description=f"Running docker image {tag}",
142	env=self._get_process_environment(env or {}),
143	immutable_input_digests=self.extra_input_digests,
144	)
145
146
147	class DockerBinary(_DockerPodmanMixin):	9✔
148	"""The `docker` binary."""
149
150
151	class PodmanBinary(_DockerPodmanMixin):	9✔
152	"""The `podman` binary."""
153
154
155	class BuildctlBinary(BaseBinary):	9✔
156	"""The `buildctl` binary."""
157
158	def build_image(	9✔
159	self,
160	tags: tuple[str, ...],
161	digest: Digest,
162	dockerfile: str,
163	build_args: DockerBuildArgs,
164	context_root: str,
165	env: Mapping[str, str],
166	extra_args: tuple[str, ...] = (),
167	) -> Process:
NEW 168	args = [	×
169	self.path,
170	"build",
171	"--frontend",
172	"dockerfile.v0",
173	"--local",
174	f"context={context_root}",
175	"--local",
176	f"dockerfile={os.path.dirname(dockerfile)}",
177	"--opt",
178	f"filename={os.path.basename(dockerfile)}",
179	*extra_args,
180	]
181
NEW 182	for build_arg in build_args:	×
NEW 183	args.extend(["--opt", f"build-arg:{build_arg}"])	×
184
NEW 185	for tag in tags:	×
NEW 186	args.extend(["--output", f"type=image,name={tag},push=true"])	×
187
NEW 188	return Process(	×
189	argv=tuple(args),
190	description=(
191	f"Building docker image {tags[0]}"
192	+ (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
193	),
194	env=self._get_process_environment(env),
195	input_digest=digest,
196	immutable_input_digests=self.extra_input_digests,
197	cache_scope=ProcessCacheScope.PER_SESSION,
198	)
199
200
201	async def _get_docker_tools_shims(	9✔
202	*,
203	tools: Sequence[str],
204	optional_tools: Sequence[str],
205	search_path: Sequence[str],
206	rationale: str,
207	) -> BinaryShims:
208	all_binary_first_paths: list[BinaryPath] = []	1✔
209
210	if tools:	1✔
211	tools_requests = [	1✔
212	BinaryPathRequest(binary_name=binary_name, search_path=search_path)
213	for binary_name in tools
214	]
215
216	tools_paths = await concurrently(	1✔
217	find_binary(tools_request, **implicitly()) for tools_request in tools_requests
218	)
219
220	all_binary_first_paths.extend(	1✔
221	[
222	path.first_path_or_raise(request, rationale=rationale)
223	for request, path in zip(tools_requests, tools_paths)
224	]
225	)
226
227	if optional_tools:	1✔
228	optional_tools_requests = [	1✔
229	BinaryPathRequest(binary_name=binary_name, search_path=search_path)
230	for binary_name in optional_tools
231	]
232
233	optional_tools_paths = await concurrently(	1✔
234	find_binary(optional_tools_request, **implicitly())
235	for optional_tools_request in optional_tools_requests
236	)
237
238	all_binary_first_paths.extend(	1✔
239	[
240	cast(BinaryPath, path.first_path) # safe since we check for non-empty paths below
241	for path in optional_tools_paths
242	if path.paths
243	]
244	)
245
246	tools_shims = await create_binary_shims(	1✔
247	BinaryShimsRequest.for_paths(
248	*all_binary_first_paths,
249	rationale=rationale,
250	),
251	**implicitly(),
252	)
253
254	return tools_shims	1✔
255
256
257	async def get_binary(	9✔
258	binary_name: str,
259	binary_cls: type[T],
260	docker_options: DockerOptions,
261	docker_options_env_aware: DockerOptions.EnvironmentAware,
262	) -> T:
263	search_path = docker_options_env_aware.executable_search_path	4✔
264
265	request = BinaryPathRequest(	4✔
266	binary_name=binary_name,
267	search_path=search_path,
268	test=BinaryPathTest(args=["-v"]),
269	)
270	paths = await find_binary(request, **implicitly())	4✔
271	first_path = paths.first_path_or_raise(request, rationale="interact with the docker daemon")	4✔
272
273	if not docker_options.tools and not docker_options.optional_tools:	4✔
274	return binary_cls(first_path.path, first_path.fingerprint)	4✔
275
276	tools_shims = await _get_docker_tools_shims(	1✔
277	tools=docker_options.tools,
278	optional_tools=docker_options.optional_tools,
279	search_path=search_path,
280	rationale=f"use {binary_name}",
281	)
282	return binary_cls(	1✔
283	first_path.path,
284	first_path.fingerprint,
285	extra_env={"PATH": tools_shims.path_component},
286	extra_input_digests=tools_shims.immutable_input_digests,
287	)
288
289
290	@rule(desc="Finding the `docker` binary and related tooling", level=LogLevel.DEBUG)	9✔
291	async def get_docker(	9✔
292	docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
293	) -> DockerBinary:
294	return await get_binary("docker", DockerBinary, docker_options, docker_options_env_aware)	4✔
295
296
297	@rule(desc="Finding the `podman` binary and related tooling", level=LogLevel.DEBUG)	9✔
298	async def get_podman(	9✔
299	docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
300	) -> PodmanBinary:
NEW 301	return await get_binary("podman", PodmanBinary, docker_options, docker_options_env_aware)	×
302
303
304	@rule(desc="Finding the `buildctl` binary and related tooling", level=LogLevel.DEBUG)	9✔
305	async def get_buildctl(	9✔
306	docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
307	) -> BuildctlBinary:
NEW 308	return await get_binary("buildctl", BuildctlBinary, docker_options, docker_options_env_aware)	×
309
310
311	def rules():	9✔
312	return collect_rules()	9✔

pantsbuild / pants / 23505168284

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous