23616668936

Committed 26 Mar 2026 08:33PM UTC coverage: 92.848% (-0.08%) from 92.923%

Build # 23616668936

Build Type

Pull #23133

github

Committed by

web-flow

Commit Message

Merge 51b4d6d01 into 9b3c1562e

Pull Request Pull Request #23133: Add buildctl engine

Coverage Stats

325 of 386 new or added lines in 14 files covered. (84.2%)

30 existing lines in 4 files now uncovered.

91645 of 98704 relevant lines covered (92.85%)

4.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.0

/src/python/pants/backend/docker/util_rules/binaries.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
import os
from abc import ABC
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Protocol, TypeVar, cast

from pants.backend.docker.subsystems.docker_options import DockerOptions
from pants.backend.docker.util_rules.docker_build_args import DockerBuildArgs
from pants.core.util_rules.system_binaries import (
    BinaryPath,
    BinaryPathRequest,
    BinaryPathTest,
    BinaryShims,
    BinaryShimsRequest,
    create_binary_shims,
    find_binary,
)
from pants.engine.fs import Digest
from pants.engine.process import Process, ProcessCacheScope
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
from pants.util.logging import LogLevel
from pants.util.strutil import pluralize

T = TypeVar("T", bound="BaseBinary")


@dataclass(frozen=True)
class BaseBinary(BinaryPath, ABC):
    """Base class for all binary paths."""

    extra_env: Mapping[str, str]
    extra_input_digests: Mapping[str, Digest] | None

    def __init__(
        self,
        path: str,
        fingerprint: str | None = None,
        extra_env: Mapping[str, str] | None = None,
        extra_input_digests: Mapping[str, Digest] | None = None,
    ) -> None:
        object.__setattr__(self, "extra_env", {} if extra_env is None else extra_env)
        object.__setattr__(self, "extra_input_digests", extra_input_digests)
        super().__init__(path, fingerprint)

    def _get_process_environment(self, env: Mapping[str, str]) -> Mapping[str, str]:
        if not self.extra_env:
            return env

        res = {**self.extra_env, **env}

        # Merge the PATH entries, in case they are present in both `env` and `self.extra_env`.
        res["PATH"] = os.pathsep.join(
            p for p in (m.get("PATH") for m in (self.extra_env, env)) if p
        )
        return res


class BuildBinaryProtocol(Protocol):
    def build_image(
        self,
        tags: tuple[str, ...],
        digest: Digest,
        dockerfile: str,
        build_args: DockerBuildArgs,
        context_root: str,
        env: Mapping[str, str],
        output: dict[str, str] | None,
        extra_args: tuple[str, ...] = (),
        is_publish: bool = False,
    ) -> Process: ...


class PushBinaryProtocol(Protocol):
    def push_image(self, tag: str, env: Mapping[str, str] | None = None) -> Process: ...


def _comma_sep_dict_args(d: Mapping[str, str]) -> str:
    return ",".join(f"{k}={v}" for k, v in d.items())


class _DockerPodmanMixin(BaseBinary):
    def build_image(
        self,
        tags: tuple[str, ...],
        digest: Digest,
        dockerfile: str,
        build_args: DockerBuildArgs,
        context_root: str,
        env: Mapping[str, str],
        output: Mapping[str, str] | None,
        extra_args: tuple[str, ...] = (),
        is_publish: bool = False,
    ) -> Process:
        args = [self.path, "build", *extra_args]

        if output:
            args.extend(["--output", _comma_sep_dict_args(output)])

        for tag in tags:
            args.extend(["--tag", tag])

        for build_arg in build_args:
            args.extend(["--build-arg", build_arg])

        args.extend(["--file", dockerfile])

        # Docker context root.
        args.append(context_root)

        return Process(
            argv=tuple(args),
            description=(
                f"Building docker image {tags[0]}"
                + (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
            ),
            env=self._get_process_environment(env),
            input_digest=digest,
            immutable_input_digests=self.extra_input_digests,
            # We must run the docker build commands every time, even if nothing has changed,
            # in case the user ran `docker image rm` outside of Pants.
            cache_scope=ProcessCacheScope.PER_SESSION,
        )

    def push_image(self, tag: str, env: Mapping[str, str] | None = None) -> Process:
        return Process(
            argv=(self.path, "push", tag),
            cache_scope=ProcessCacheScope.PER_SESSION,
            description=f"Pushing docker image {tag}",
            env=self._get_process_environment(env or {}),
            immutable_input_digests=self.extra_input_digests,
        )

    def run_image(
        self,
        tag: str,
        *,
        docker_run_args: tuple[str, ...] | None = None,
        image_args: tuple[str, ...] | None = None,
        env: Mapping[str, str] | None = None,
    ) -> Process:
        return Process(
            argv=(
                self.path,
                "run",
                *(docker_run_args or []),
                tag,
                *(image_args or []),
            ),
            cache_scope=ProcessCacheScope.PER_SESSION,
            description=f"Running docker image {tag}",
            env=self._get_process_environment(env or {}),
            immutable_input_digests=self.extra_input_digests,
        )


class DockerBinary(_DockerPodmanMixin):
    """The `docker` binary."""


class PodmanBinary(_DockerPodmanMixin):
    """The `podman` binary."""


class BuildctlBinary(BaseBinary):
    """The `buildctl` binary."""

    @staticmethod
    def _special_output_handling(output: Mapping[str, str] | None) -> bool:
        if output:
            try:
                output_type = output["type"]
            except KeyError as ke:
                raise ValueError(
                    "docker_image output type field is required when specifying output"
                ) from ke
            return output_type == "image" and "name" not in output
        return True

    def build_image(
        self,
        tags: tuple[str, ...],
        digest: Digest,
        dockerfile: str,
        build_args: DockerBuildArgs,
        context_root: str,
        env: Mapping[str, str],
        output: Mapping[str, str] | None,
        extra_args: tuple[str, ...] = (),
        is_publish: bool = False,
    ) -> Process:
        args = [
            self.path,
            "build",
            "--frontend",
            "dockerfile.v0",
            "--local",
            f"context={context_root}",
            "--local",
            f"dockerfile={os.path.dirname(dockerfile)}",
            "--opt",
            f"filename={os.path.basename(dockerfile)}",
            *extra_args,
        ]

        for build_arg in build_args:
            args.extend(["--opt", f"build-arg:{build_arg}"])

        if self._special_output_handling(output):
            publish_suffix = ",push=true" if is_publish else ""
            for tag in tags:
                args.extend(["--output", f"type=image,name={tag}{publish_suffix}"])
        else:
            args.extend(["--output", _comma_sep_dict_args(cast(Mapping[str, str], output))])

        return Process(
            argv=tuple(args),
            description=(
                f"Building docker image {tags[0]}"
                + (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
            ),
            env=self._get_process_environment(env),
            input_digest=digest,
            immutable_input_digests=self.extra_input_digests,
            cache_scope=ProcessCacheScope.PER_SESSION,
        )


async def _get_docker_tools_shims(
    *,
    tools: Sequence[str],
    optional_tools: Sequence[str],
    search_path: Sequence[str],
    rationale: str,
) -> BinaryShims:
    all_binary_first_paths: list[BinaryPath] = []

    if tools:
        tools_requests = [
            BinaryPathRequest(binary_name=binary_name, search_path=search_path)
            for binary_name in tools
        ]

        tools_paths = await concurrently(
            find_binary(tools_request, **implicitly()) for tools_request in tools_requests
        )

        all_binary_first_paths.extend(
            [
                path.first_path_or_raise(request, rationale=rationale)
                for request, path in zip(tools_requests, tools_paths)
            ]
        )

    if optional_tools:
        optional_tools_requests = [
            BinaryPathRequest(binary_name=binary_name, search_path=search_path)
            for binary_name in optional_tools
        ]

        optional_tools_paths = await concurrently(
            find_binary(optional_tools_request, **implicitly())
            for optional_tools_request in optional_tools_requests
        )

        all_binary_first_paths.extend(
            [
                cast(BinaryPath, path.first_path)  # safe since we check for non-empty paths below
                for path in optional_tools_paths
                if path.paths
            ]
        )

    tools_shims = await create_binary_shims(
        BinaryShimsRequest.for_paths(
            *all_binary_first_paths,
            rationale=rationale,
        ),
        **implicitly(),
    )

    return tools_shims


async def get_binary(
    binary_name: str,
    binary_cls: type[T],
    docker_options: DockerOptions,
    docker_options_env_aware: DockerOptions.EnvironmentAware,
) -> T:
    search_path = docker_options_env_aware.executable_search_path

    request = BinaryPathRequest(
        binary_name=binary_name,
        search_path=search_path,
        test=BinaryPathTest(args=["-v"]),
    )
    paths = await find_binary(request, **implicitly())
    first_path = paths.first_path_or_raise(request, rationale="interact with the docker daemon")

    if not docker_options.tools and not docker_options.optional_tools:
        return binary_cls(first_path.path, first_path.fingerprint)

    tools_shims = await _get_docker_tools_shims(
        tools=docker_options.tools,
        optional_tools=docker_options.optional_tools,
        search_path=search_path,
        rationale=f"use {binary_name}",
    )
    return binary_cls(
        first_path.path,
        first_path.fingerprint,
        extra_env={"PATH": tools_shims.path_component},
        extra_input_digests=tools_shims.immutable_input_digests,
    )


@rule(desc="Finding the `docker` binary and related tooling", level=LogLevel.DEBUG)
async def get_docker(
    docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
) -> DockerBinary:
    return await get_binary("docker", DockerBinary, docker_options, docker_options_env_aware)


@rule(desc="Finding the `podman` binary and related tooling", level=LogLevel.DEBUG)
async def get_podman(
    docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
) -> PodmanBinary:
    return await get_binary("podman", PodmanBinary, docker_options, docker_options_env_aware)


@rule(desc="Finding the `buildctl` binary and related tooling", level=LogLevel.DEBUG)
async def get_buildctl(
    docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
) -> BuildctlBinary:
    return await get_binary("buildctl", BuildctlBinary, docker_options, docker_options_env_aware)


def rules():
    return collect_rules()

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	import os	9✔
4	from abc import ABC	9✔
5	from collections.abc import Mapping, Sequence	9✔
6	from dataclasses import dataclass	9✔
7	from typing import Protocol, TypeVar, cast	9✔
8
9	from pants.backend.docker.subsystems.docker_options import DockerOptions	9✔
10	from pants.backend.docker.util_rules.docker_build_args import DockerBuildArgs	9✔
11	from pants.core.util_rules.system_binaries import (	9✔
12	BinaryPath,
13	BinaryPathRequest,
14	BinaryPathTest,
15	BinaryShims,
16	BinaryShimsRequest,
17	create_binary_shims,
18	find_binary,
19	)
20	from pants.engine.fs import Digest	9✔
21	from pants.engine.process import Process, ProcessCacheScope	9✔
22	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	9✔
23	from pants.util.logging import LogLevel	9✔
24	from pants.util.strutil import pluralize	9✔
25
26	T = TypeVar("T", bound="BaseBinary")	9✔
27
28
29	@dataclass(frozen=True)	9✔
30	class BaseBinary(BinaryPath, ABC):	9✔
31	"""Base class for all binary paths."""
32
33	extra_env: Mapping[str, str]
34	extra_input_digests: Mapping[str, Digest] \| None
35
36	def __init__(	9✔
37	self,
38	path: str,
39	fingerprint: str \| None = None,
40	extra_env: Mapping[str, str] \| None = None,
41	extra_input_digests: Mapping[str, Digest] \| None = None,
42	) -> None:
43	object.__setattr__(self, "extra_env", {} if extra_env is None else extra_env)	5✔
44	object.__setattr__(self, "extra_input_digests", extra_input_digests)	5✔
45	super().__init__(path, fingerprint)	5✔
46
47	def _get_process_environment(self, env: Mapping[str, str]) -> Mapping[str, str]:	9✔
48	if not self.extra_env:	5✔
49	return env	5✔
50
51	res = {self.extra_env, env}	×
52
53	# Merge the PATH entries, in case they are present in both `env` and `self.extra_env`.
54	res["PATH"] = os.pathsep.join(	×
55	p for p in (m.get("PATH") for m in (self.extra_env, env)) if p
56	)
57	return res	×
58
59
60	class BuildBinaryProtocol(Protocol):	9✔
61	def build_image(
62	self,
63	tags: tuple[str, ...],
64	digest: Digest,
65	dockerfile: str,
66	build_args: DockerBuildArgs,
67	context_root: str,
68	env: Mapping[str, str],
69	output: dict[str, str] \| None,
70	extra_args: tuple[str, ...] = (),
71	is_publish: bool = False,
72	) -> Process: ...
73
74
75	class PushBinaryProtocol(Protocol):	9✔
76	def push_image(self, tag: str, env: Mapping[str, str] \| None = None) -> Process: ...
77
78
79	def _comma_sep_dict_args(d: Mapping[str, str]) -> str:	9✔
80	return ",".join(f"{k}={v}" for k, v in d.items())	1✔
81
82
83	class _DockerPodmanMixin(BaseBinary):	9✔
84	def build_image(	9✔
85	self,
86	tags: tuple[str, ...],
87	digest: Digest,
88	dockerfile: str,
89	build_args: DockerBuildArgs,
90	context_root: str,
91	env: Mapping[str, str],
92	output: Mapping[str, str] \| None,
93	extra_args: tuple[str, ...] = (),
94	is_publish: bool = False,
95	) -> Process:
96	args = [self.path, "build", *extra_args]	4✔
97
98	if output:	4✔
99	args.extend(["--output", _comma_sep_dict_args(output)])	1✔
100
101	for tag in tags:	4✔
102	args.extend(["--tag", tag])	4✔
103
104	for build_arg in build_args:	4✔
105	args.extend(["--build-arg", build_arg])	2✔
106
107	args.extend(["--file", dockerfile])	4✔
108
109	# Docker context root.
110	args.append(context_root)	4✔
111
112	return Process(	4✔
113	argv=tuple(args),
114	description=(
115	f"Building docker image {tags[0]}"
116	+ (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
117	),
118	env=self._get_process_environment(env),
119	input_digest=digest,
120	immutable_input_digests=self.extra_input_digests,
121	# We must run the docker build commands every time, even if nothing has changed,
122	# in case the user ran `docker image rm` outside of Pants.
123	cache_scope=ProcessCacheScope.PER_SESSION,
124	)
125
126	def push_image(self, tag: str, env: Mapping[str, str] \| None = None) -> Process:	9✔
127	return Process(	2✔
128	argv=(self.path, "push", tag),
129	cache_scope=ProcessCacheScope.PER_SESSION,
130	description=f"Pushing docker image {tag}",
131	env=self._get_process_environment(env or {}),
132	immutable_input_digests=self.extra_input_digests,
133	)
134
135	def run_image(	9✔
136	self,
137	tag: str,
138	*,
139	docker_run_args: tuple[str, ...] \| None = None,
140	image_args: tuple[str, ...] \| None = None,
141	env: Mapping[str, str] \| None = None,
142	) -> Process:
143	return Process(	1✔
144	argv=(
145	self.path,
146	"run",
147	*(docker_run_args or []),
148	tag,
149	*(image_args or []),
150	),
151	cache_scope=ProcessCacheScope.PER_SESSION,
152	description=f"Running docker image {tag}",
153	env=self._get_process_environment(env or {}),
154	immutable_input_digests=self.extra_input_digests,
155	)
156
157
158	class DockerBinary(_DockerPodmanMixin):	9✔
159	"""The `docker` binary."""
160
161
162	class PodmanBinary(_DockerPodmanMixin):	9✔
163	"""The `podman` binary."""
164
165
166	class BuildctlBinary(BaseBinary):	9✔
167	"""The `buildctl` binary."""
168
169	@staticmethod	9✔
170	def _special_output_handling(output: Mapping[str, str] \| None) -> bool:	9✔
NEW 171	if output:	×
NEW 172	try:	×
NEW 173	output_type = output["type"]	×
NEW 174	except KeyError as ke:	×
NEW 175	raise ValueError(	×
176	"docker_image output type field is required when specifying output"
177	) from ke
NEW 178	return output_type == "image" and "name" not in output	×
NEW 179	return True	×
180
181	def build_image(	9✔
182	self,
183	tags: tuple[str, ...],
184	digest: Digest,
185	dockerfile: str,
186	build_args: DockerBuildArgs,
187	context_root: str,
188	env: Mapping[str, str],
189	output: Mapping[str, str] \| None,
190	extra_args: tuple[str, ...] = (),
191	is_publish: bool = False,
192	) -> Process:
NEW 193	args = [	×
194	self.path,
195	"build",
196	"--frontend",
197	"dockerfile.v0",
198	"--local",
199	f"context={context_root}",
200	"--local",
201	f"dockerfile={os.path.dirname(dockerfile)}",
202	"--opt",
203	f"filename={os.path.basename(dockerfile)}",
204	*extra_args,
205	]
206
NEW 207	for build_arg in build_args:	×
NEW 208	args.extend(["--opt", f"build-arg:{build_arg}"])	×
209
NEW 210	if self._special_output_handling(output):	×
NEW 211	publish_suffix = ",push=true" if is_publish else ""	×
NEW 212	for tag in tags:	×
NEW 213	args.extend(["--output", f"type=image,name={tag}{publish_suffix}"])	×
214	else:
NEW 215	args.extend(["--output", _comma_sep_dict_args(cast(Mapping[str, str], output))])	×
216
NEW 217	return Process(	×
218	argv=tuple(args),
219	description=(
220	f"Building docker image {tags[0]}"
221	+ (f" +{pluralize(len(tags) - 1, 'additional tag')}." if len(tags) > 1 else "")
222	),
223	env=self._get_process_environment(env),
224	input_digest=digest,
225	immutable_input_digests=self.extra_input_digests,
226	cache_scope=ProcessCacheScope.PER_SESSION,
227	)
228
229
230	async def _get_docker_tools_shims(	9✔
231	*,
232	tools: Sequence[str],
233	optional_tools: Sequence[str],
234	search_path: Sequence[str],
235	rationale: str,
236	) -> BinaryShims:
237	all_binary_first_paths: list[BinaryPath] = []	1✔
238
239	if tools:	1✔
240	tools_requests = [	1✔
241	BinaryPathRequest(binary_name=binary_name, search_path=search_path)
242	for binary_name in tools
243	]
244
245	tools_paths = await concurrently(	1✔
246	find_binary(tools_request, **implicitly()) for tools_request in tools_requests
247	)
248
249	all_binary_first_paths.extend(	1✔
250	[
251	path.first_path_or_raise(request, rationale=rationale)
252	for request, path in zip(tools_requests, tools_paths)
253	]
254	)
255
256	if optional_tools:	1✔
257	optional_tools_requests = [	1✔
258	BinaryPathRequest(binary_name=binary_name, search_path=search_path)
259	for binary_name in optional_tools
260	]
261
262	optional_tools_paths = await concurrently(	1✔
263	find_binary(optional_tools_request, **implicitly())
264	for optional_tools_request in optional_tools_requests
265	)
266
267	all_binary_first_paths.extend(	1✔
268	[
269	cast(BinaryPath, path.first_path) # safe since we check for non-empty paths below
270	for path in optional_tools_paths
271	if path.paths
272	]
273	)
274
275	tools_shims = await create_binary_shims(	1✔
276	BinaryShimsRequest.for_paths(
277	*all_binary_first_paths,
278	rationale=rationale,
279	),
280	**implicitly(),
281	)
282
283	return tools_shims	1✔
284
285
286	async def get_binary(	9✔
287	binary_name: str,
288	binary_cls: type[T],
289	docker_options: DockerOptions,
290	docker_options_env_aware: DockerOptions.EnvironmentAware,
291	) -> T:
292	search_path = docker_options_env_aware.executable_search_path	4✔
293
294	request = BinaryPathRequest(	4✔
295	binary_name=binary_name,
296	search_path=search_path,
297	test=BinaryPathTest(args=["-v"]),
298	)
299	paths = await find_binary(request, **implicitly())	4✔
300	first_path = paths.first_path_or_raise(request, rationale="interact with the docker daemon")	4✔
301
302	if not docker_options.tools and not docker_options.optional_tools:	4✔
303	return binary_cls(first_path.path, first_path.fingerprint)	4✔
304
305	tools_shims = await _get_docker_tools_shims(	1✔
306	tools=docker_options.tools,
307	optional_tools=docker_options.optional_tools,
308	search_path=search_path,
309	rationale=f"use {binary_name}",
310	)
311	return binary_cls(	1✔
312	first_path.path,
313	first_path.fingerprint,
314	extra_env={"PATH": tools_shims.path_component},
315	extra_input_digests=tools_shims.immutable_input_digests,
316	)
317
318
319	@rule(desc="Finding the `docker` binary and related tooling", level=LogLevel.DEBUG)	9✔
320	async def get_docker(	9✔
321	docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
322	) -> DockerBinary:
323	return await get_binary("docker", DockerBinary, docker_options, docker_options_env_aware)	4✔
324
325
326	@rule(desc="Finding the `podman` binary and related tooling", level=LogLevel.DEBUG)	9✔
327	async def get_podman(	9✔
328	docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
329	) -> PodmanBinary:
NEW 330	return await get_binary("podman", PodmanBinary, docker_options, docker_options_env_aware)	×
331
332
333	@rule(desc="Finding the `buildctl` binary and related tooling", level=LogLevel.DEBUG)	9✔
334	async def get_buildctl(	9✔
335	docker_options: DockerOptions, docker_options_env_aware: DockerOptions.EnvironmentAware
336	) -> BuildctlBinary:
NEW 337	return await get_binary("buildctl", BuildctlBinary, docker_options, docker_options_env_aware)	×
338
339
340	def rules():	9✔
341	return collect_rules()	9✔

pantsbuild / pants / 23616668936

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous