18517631058

Committed 15 Oct 2025 04:18AM UTC coverage: 69.207% (-11.1%) from 80.267%

Build # 18517631058

Build Type

Pull #22745

github

Committed by

web-flow

Commit Message

Merge 642a76ca1 into 99919310e

Pull Request Pull Request #22745: [windows] Add windows support in the stdio crate.

Run Details

53815 of 77759 relevant lines covered (69.21%)

2.42 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

44.9

/src/python/pants/backend/docker/util_rules/docker_build_context.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import logging
import re
import shlex
from abc import ABC
from collections.abc import Iterable, Mapping
from dataclasses import dataclass

from pants.backend.docker.package_types import BuiltDockerImage
from pants.backend.docker.subsystems.docker_options import DockerOptions
from pants.backend.docker.subsystems.dockerfile_parser import (
    DockerfileInfo,
    DockerfileInfoRequest,
    parse_dockerfile,
)
from pants.backend.docker.target_types import DockerImageSourceField
from pants.backend.docker.util_rules.docker_build_args import (
    DockerBuildArgs,
    DockerBuildArgsRequest,
    docker_build_args,
)
from pants.backend.docker.util_rules.docker_build_env import (
    DockerBuildEnvironment,
    DockerBuildEnvironmentError,
    DockerBuildEnvironmentRequest,
    docker_build_environment_vars,
)
from pants.backend.docker.utils import image_ref_regexp, suggest_renames
from pants.backend.docker.value_interpolation import DockerBuildArgsInterpolationValue
from pants.backend.shell.target_types import ShellSourceField
from pants.core.goals.package import (
    BuiltPackage,
    EnvironmentAwarePackageRequest,
    PackageFieldSet,
    environment_aware_package,
)
from pants.core.target_types import FileSourceField
from pants.core.util_rules.source_files import SourceFilesRequest, determine_source_files
from pants.engine.addresses import Address, UnparsedAddressInputs
from pants.engine.fs import Digest, MergeDigests, Snapshot
from pants.engine.internals.graph import (
    find_valid_field_sets,
    resolve_targets,
    resolve_unparsed_address_inputs,
)
from pants.engine.internals.graph import transitive_targets as transitive_targets_get
from pants.engine.intrinsics import digest_to_snapshot
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
from pants.engine.target import (
    Dependencies,
    DependenciesRequest,
    FieldSetsPerTargetRequest,
    GeneratedSources,
    GenerateSourcesRequest,
    SourcesField,
    TransitiveTargetsRequest,
)
from pants.engine.unions import UnionRule
from pants.util.strutil import softwrap, stable_hash
from pants.util.value_interpolation import InterpolationContext, InterpolationValue

logger = logging.getLogger(__name__)


class DockerBuildContextError(Exception):
    pass


class DockerContextFilesAcceptableInputsField(ABC, SourcesField):
    """This is a meta field for the context files generator, to tell the codegen machinery what
    source fields are good to use as-is.

    Use `DockerContextFilesAcceptableInputsField.register(<SourceField>)` to register input fields
    that should be accepted.

    This is implemented using the `ABC.register` from Python lib:
    https://docs.python.org/3/library/abc.html#abc.ABCMeta.register
    """


# These sources will be used to populate the build context as-is.
DockerContextFilesAcceptableInputsField.register(ShellSourceField)


class DockerContextFilesSourcesField(SourcesField):
    """This is just a type marker for the codegen machinery."""


class GenerateDockerContextFiles(GenerateSourcesRequest):
    """This translates all files from acceptable Source fields for the docker context using the
    `codegen` machinery."""

    input = DockerContextFilesAcceptableInputsField
    output = DockerContextFilesSourcesField
    exportable = False


@rule
async def hydrate_input_sources(request: GenerateDockerContextFiles) -> GeneratedSources:
    # We simply pass the files on, as-is
    return GeneratedSources(request.protocol_sources)


@dataclass(frozen=True)
class DockerBuildContextRequest:
    address: Address
    build_upstream_images: bool = False


@dataclass(frozen=True)
class DockerBuildContext:
    build_args: DockerBuildArgs
    digest: Digest
    build_env: DockerBuildEnvironment
    upstream_image_ids: tuple[str, ...]
    dockerfile: str
    interpolation_context: InterpolationContext
    copy_source_vs_context_source: tuple[tuple[str, str], ...]
    stages: tuple[str, ...]

    @classmethod
    def create(
        cls,
        build_args: DockerBuildArgs,
        snapshot: Snapshot,
        build_env: DockerBuildEnvironment,
        upstream_image_ids: Iterable[str],
        dockerfile_info: DockerfileInfo,
        should_suggest_renames: bool = True,
    ) -> DockerBuildContext:
        interpolation_context: dict[str, dict[str, str] | InterpolationValue] = {}

        if build_args:
            interpolation_context["build_args"] = cls._merge_build_args(
                dockerfile_info, build_args, build_env
            )

        # Override default value type for the `build_args` context to get helpful error messages.
        interpolation_context["build_args"] = DockerBuildArgsInterpolationValue(
            interpolation_context.get("build_args", {})
        )

        # Data from Pants.
        interpolation_context["pants"] = {
            # Present hash for all inputs that can be used for image tagging.
            "hash": stable_hash((build_args, build_env, snapshot.digest)),
        }

        # Base image tags values for all stages (as parsed from the Dockerfile instructions).
        stage_names, tags_values = cls._get_stages_and_tags(
            dockerfile_info, interpolation_context["build_args"]
        )
        interpolation_context["tags"] = tags_values

        copy_source_vs_context_source = (
            tuple(
                suggest_renames(
                    tentative_paths=(
                        # We don't want to include the Dockerfile as a suggested rename
                        dockerfile_info.source,
                        *dockerfile_info.copy_source_paths,
                    ),
                    actual_files=snapshot.files,
                    actual_dirs=snapshot.dirs,
                )
            )
            if should_suggest_renames
            else ()
        )

        return cls(
            build_args=build_args,
            digest=snapshot.digest,
            dockerfile=dockerfile_info.source,
            build_env=build_env,
            upstream_image_ids=tuple(sorted(upstream_image_ids)),
            interpolation_context=InterpolationContext.from_dict(interpolation_context),
            copy_source_vs_context_source=copy_source_vs_context_source,
            stages=tuple(sorted(stage_names)),
        )

    @classmethod
    def _get_stages_and_tags(
        cls, dockerfile_info: DockerfileInfo, build_args: Mapping[str, str]
    ) -> tuple[set[str], dict[str, str]]:
        # Go over all FROM tags and names for all stages.
        stage_names: set[str] = set()
        # tag is empty if image is referenced by digest instead
        stage_tags = ([*tag.split(maxsplit=1), ""][:2] for tag in dockerfile_info.version_tags)
        tags_values: dict[str, str] = {}
        for idx, (stage, tag) in enumerate(stage_tags):
            if tag.startswith("build-arg:"):
                build_arg = tag[10:]
                image_ref = build_args.get(build_arg) or dockerfile_info.build_args.to_dict().get(
                    build_arg
                )
                if not image_ref:
                    raise DockerBuildContextError(
                        f"Failed to parse Dockerfile baseimage tag for stage {stage} in "
                        f"{dockerfile_info.address} target, unknown build ARG: {build_arg!r}."
                    )
                parsed = re.match(image_ref_regexp, image_ref.strip("\"'"))
                tag = parsed.group("tag") or (parsed.group("digest") and "latest") if parsed else ""
                if not tag:
                    raise DockerBuildContextError(
                        f"Failed to parse Dockerfile baseimage tag for stage {stage} in "
                        f"{dockerfile_info.address} target, from image ref: {image_ref}."
                    )

            if stage != f"stage{idx}":
                stage_names.add(stage)
            if tag:
                if idx == 0:
                    # Expose the first (stage0) FROM directive as the "baseimage".
                    tags_values["baseimage"] = tag
                tags_values[stage] = tag

        return stage_names, tags_values

    @staticmethod
    def _merge_build_args(
        dockerfile_info: DockerfileInfo,
        build_args: DockerBuildArgs,
        build_env: DockerBuildEnvironment,
    ) -> dict[str, str]:
        # Extract default arg values from the parsed Dockerfile.
        build_arg_defaults = {
            def_name: def_value
            for def_name, has_default, def_value in [
                def_arg.partition("=") for def_arg in dockerfile_info.build_args
            ]
            if has_default
        }
        try:
            # Create build args context value, based on defined build_args and
            # extra_build_args. We do _not_ auto "magically" pick up all ARG names from the
            # Dockerfile as first class args to use as placeholders, to make it more explicit
            # which args are actually being used by Pants. We do pick up any defined default ARG
            # values from the Dockerfile however, in order to not having to duplicate them in
            # the BUILD files.
            return {
                arg_name: (
                    arg_value
                    if has_value
                    else build_env.get(arg_name, build_arg_defaults.get(arg_name))
                )
                for arg_name, has_value, arg_value in [
                    build_arg.partition("=") for build_arg in build_args
                ]
            }
        except DockerBuildEnvironmentError as e:
            raise DockerBuildContextError(
                f"Undefined value for build arg on the {dockerfile_info.address} target: {e}"
                "\n\nIf you did not intend to inherit the value for this build arg from the "
                "environment, provide a default value with the option `[docker].build_args` "
                "or in the `extra_build_args` field on the target definition. Alternatively, "
                "you may also provide a default value on the `ARG` instruction directly in "
                "the `Dockerfile`."
            ) from e


@rule
async def create_docker_build_context(
    request: DockerBuildContextRequest,
    options: DockerOptions,
) -> DockerBuildContext:
    # Get all targets to include in context.
    transitive_targets = await transitive_targets_get(
        TransitiveTargetsRequest([request.address]), **implicitly()
    )
    docker_image = transitive_targets.roots[0]

    # Get all dependencies for the root target.
    root_dependencies = await resolve_targets(
        **implicitly(DependenciesRequest(docker_image.get(Dependencies)))
    )

    # Get all file sources from the root dependencies. That includes any non-file sources that can
    # be "codegen"ed into a file source.
    sources_request = determine_source_files(
        SourceFilesRequest(
            sources_fields=[tgt.get(SourcesField) for tgt in root_dependencies],
            for_sources_types=(
                DockerContextFilesSourcesField,
                FileSourceField,
            ),
            enable_codegen=True,
        )
    )

    embedded_pkgs_per_target_request = find_valid_field_sets(
        FieldSetsPerTargetRequest(PackageFieldSet, transitive_targets.dependencies), **implicitly()
    )

    sources, embedded_pkgs_per_target, dockerfile_info = await concurrently(
        sources_request,
        embedded_pkgs_per_target_request,
        parse_dockerfile(DockerfileInfoRequest(docker_image.address), **implicitly()),
    )

    # Package binary dependencies for build context.
    pkgs_wanting_embedding = [
        field_set
        for field_set in embedded_pkgs_per_target.field_sets
        # Exclude docker images, unless build_upstream_images is true.
        if (
            request.build_upstream_images
            or not isinstance(getattr(field_set, "source", None), DockerImageSourceField)
        )
    ]
    embedded_pkgs = await concurrently(
        environment_aware_package(EnvironmentAwarePackageRequest(field_set))
        for field_set in pkgs_wanting_embedding
    )

    if request.build_upstream_images:
        images_str = ", ".join(
            a.tags[0] for p in embedded_pkgs for a in p.artifacts if isinstance(a, BuiltDockerImage)
        )
        if images_str:
            logger.debug(f"Built upstream Docker images: {images_str}")
        else:
            logger.debug("Did not build any upstream Docker images")

    packages_str = ", ".join(a.relpath for p in embedded_pkgs for a in p.artifacts if a.relpath)
    if packages_str:
        logger.debug(f"Built packages for Docker image: {packages_str}")
    else:
        logger.debug("Did not build any packages for Docker image")

    embedded_pkgs_digest = [built_package.digest for built_package in embedded_pkgs]
    all_digests = (dockerfile_info.digest, sources.snapshot.digest, *embedded_pkgs_digest)

    # Merge all digests to get the final docker build context digest.
    context_request = digest_to_snapshot(**implicitly(MergeDigests(d for d in all_digests if d)))

    # Requests for build args and env
    build_args_request = docker_build_args(DockerBuildArgsRequest(docker_image), **implicitly())
    build_env_request = docker_build_environment_vars(
        DockerBuildEnvironmentRequest(docker_image), **implicitly()
    )
    context, supplied_build_args, build_env = await concurrently(
        context_request, build_args_request, build_env_request
    )

    build_args = supplied_build_args

    upstream_image_ids = []
    if request.build_upstream_images:
        # Update build arg values for FROM image build args.

        # Get the FROM image build args with defined values in the Dockerfile & build args.
        dockerfile_build_args = dockerfile_info.from_image_build_args.with_overrides(
            supplied_build_args
        ).nonempty()
        # Parse the build args values into Address instances.
        from_image_addresses = await resolve_unparsed_address_inputs(
            UnparsedAddressInputs(
                dockerfile_build_args.values(),
                owning_address=dockerfile_info.address,
                description_of_origin=softwrap(
                    f"""
                    the FROM arguments from the file {dockerfile_info.source}
                    from the target {dockerfile_info.address}
                    """
                ),
                skip_invalid_addresses=True,
            ),
            **implicitly(),
        )
        # Map those addresses to the corresponding built image ref (tag).
        address_to_built_image_tag = {
            field_set.address: image.tags[0]
            for field_set, built in zip(embedded_pkgs_per_target.field_sets, embedded_pkgs)
            for image in built.artifacts
            if isinstance(image, BuiltDockerImage)
        }
        upstream_image_ids = [
            image.image_id
            for built in embedded_pkgs
            for image in built.artifacts
            if isinstance(image, BuiltDockerImage)
        ]
        # Create the FROM image build args.
        from_image_build_args = [
            f"{arg_name}={address_to_built_image_tag[addr]}"
            for arg_name, addr in zip(dockerfile_build_args.keys(), from_image_addresses)
        ]
        build_args = build_args.extended(from_image_build_args)

    # Render build args for turning COPY values in ARGS which are targets into their output
    dockerfile_copy_args = dockerfile_info.copy_build_args.with_overrides(
        supplied_build_args
    ).nonempty()

    def get_artifact_paths(built_package: BuiltPackage) -> list[str]:
        return [e.relpath for e in built_package.artifacts if e.relpath]

    addrs_to_paths = {
        field_set.address: get_artifact_paths(pkg)
        for field_set, pkg in zip(embedded_pkgs_per_target.field_sets, embedded_pkgs)
    }

    copy_arg_as_build_args = await fill_args_from_copy(
        dockerfile_copy_args, dockerfile_info, addrs_to_paths
    )

    build_args = build_args.extended(copy_arg_as_build_args)

    return DockerBuildContext.create(
        build_args=build_args,
        snapshot=context,
        upstream_image_ids=upstream_image_ids,
        dockerfile_info=dockerfile_info,
        build_env=build_env,
        should_suggest_renames=options.suggest_renames,
    )


async def fill_args_from_copy(
    dockerfile_copy_args: dict[str, str], dockerfile_info, addrs_to_paths
):
    copy_arg_addresses = await resolve_unparsed_address_inputs(
        UnparsedAddressInputs(
            dockerfile_info.copy_build_args.to_dict().values(),
            owning_address=dockerfile_info.address,
            description_of_origin=softwrap(
                f"""
                the COPY arguments from the file {dockerfile_info.source}
                from the target {dockerfile_info.address}
                """
            ),
            skip_invalid_addresses=True,
        ),
        **implicitly(),
    )

    def resolve_arg(arg_name, maybe_addr) -> str:
        if maybe_addr in addrs_to_paths:
            return f"{arg_name}={shlex.join(addrs_to_paths[maybe_addr])}"
        else:
            # When the ARG value is a reference to a normal file
            return f"{arg_name}={maybe_addr}"

    copy_arg_as_build_args = [
        resolve_arg(arg_name, arg_value)
        for arg_name, arg_value in (zip(dockerfile_copy_args.keys(), copy_arg_addresses))
    ]
    return copy_arg_as_build_args


def rules():
    return (
        *collect_rules(),
        UnionRule(GenerateSourcesRequest, GenerateDockerContextFiles),
    )

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	5✔
5
6	import logging	5✔
7	import re	5✔
8	import shlex	5✔
9	from abc import ABC	5✔
10	from collections.abc import Iterable, Mapping	5✔
11	from dataclasses import dataclass	5✔
12
13	from pants.backend.docker.package_types import BuiltDockerImage	5✔
14	from pants.backend.docker.subsystems.docker_options import DockerOptions	5✔
15	from pants.backend.docker.subsystems.dockerfile_parser import (	5✔
16	DockerfileInfo,
17	DockerfileInfoRequest,
18	parse_dockerfile,
19	)
20	from pants.backend.docker.target_types import DockerImageSourceField	5✔
21	from pants.backend.docker.util_rules.docker_build_args import (	5✔
22	DockerBuildArgs,
23	DockerBuildArgsRequest,
24	docker_build_args,
25	)
26	from pants.backend.docker.util_rules.docker_build_env import (	5✔
27	DockerBuildEnvironment,
28	DockerBuildEnvironmentError,
29	DockerBuildEnvironmentRequest,
30	docker_build_environment_vars,
31	)
32	from pants.backend.docker.utils import image_ref_regexp, suggest_renames	5✔
33	from pants.backend.docker.value_interpolation import DockerBuildArgsInterpolationValue	5✔
34	from pants.backend.shell.target_types import ShellSourceField	5✔
35	from pants.core.goals.package import (	5✔
36	BuiltPackage,
37	EnvironmentAwarePackageRequest,
38	PackageFieldSet,
39	environment_aware_package,
40	)
41	from pants.core.target_types import FileSourceField	5✔
42	from pants.core.util_rules.source_files import SourceFilesRequest, determine_source_files	5✔
43	from pants.engine.addresses import Address, UnparsedAddressInputs	5✔
44	from pants.engine.fs import Digest, MergeDigests, Snapshot	5✔
45	from pants.engine.internals.graph import (	5✔
46	find_valid_field_sets,
47	resolve_targets,
48	resolve_unparsed_address_inputs,
49	)
50	from pants.engine.internals.graph import transitive_targets as transitive_targets_get	5✔
51	from pants.engine.intrinsics import digest_to_snapshot	5✔
52	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	5✔
53	from pants.engine.target import (	5✔
54	Dependencies,
55	DependenciesRequest,
56	FieldSetsPerTargetRequest,
57	GeneratedSources,
58	GenerateSourcesRequest,
59	SourcesField,
60	TransitiveTargetsRequest,
61	)
62	from pants.engine.unions import UnionRule	5✔
63	from pants.util.strutil import softwrap, stable_hash	5✔
64	from pants.util.value_interpolation import InterpolationContext, InterpolationValue	5✔
65
66	logger = logging.getLogger(__name__)	5✔
67
68
69	class DockerBuildContextError(Exception):	5✔
70	pass	5✔
71
72
73	class DockerContextFilesAcceptableInputsField(ABC, SourcesField):	5✔
74	"""This is a meta field for the context files generator, to tell the codegen machinery what
75	source fields are good to use as-is.
76
77	Use `DockerContextFilesAcceptableInputsField.register(<SourceField>)` to register input fields
78	that should be accepted.
79
80	This is implemented using the `ABC.register` from Python lib:
81	https://docs.python.org/3/library/abc.html#abc.ABCMeta.register
82	"""
83
84
85	# These sources will be used to populate the build context as-is.
86	DockerContextFilesAcceptableInputsField.register(ShellSourceField)	5✔
87
88
89	class DockerContextFilesSourcesField(SourcesField):	5✔
90	"""This is just a type marker for the codegen machinery."""
91
92
93	class GenerateDockerContextFiles(GenerateSourcesRequest):	5✔
94	"""This translates all files from acceptable Source fields for the docker context using the
95	`codegen` machinery."""
96
97	input = DockerContextFilesAcceptableInputsField	5✔
98	output = DockerContextFilesSourcesField	5✔
99	exportable = False	5✔
100
101
102	@rule	5✔
103	async def hydrate_input_sources(request: GenerateDockerContextFiles) -> GeneratedSources:	5✔
104	# We simply pass the files on, as-is
105	return GeneratedSources(request.protocol_sources)	×
106
107
108	@dataclass(frozen=True)	5✔
109	class DockerBuildContextRequest:	5✔
110	address: Address	5✔
111	build_upstream_images: bool = False	5✔
112
113
114	@dataclass(frozen=True)	5✔
115	class DockerBuildContext:	5✔
116	build_args: DockerBuildArgs	5✔
117	digest: Digest	5✔
118	build_env: DockerBuildEnvironment	5✔
119	upstream_image_ids: tuple[str, ...]	5✔
120	dockerfile: str	5✔
121	interpolation_context: InterpolationContext	5✔
122	copy_source_vs_context_source: tuple[tuple[str, str], ...]	5✔
123	stages: tuple[str, ...]	5✔
124
125	@classmethod	5✔
126	def create(	5✔
127	cls,
128	build_args: DockerBuildArgs,
129	snapshot: Snapshot,
130	build_env: DockerBuildEnvironment,
131	upstream_image_ids: Iterable[str],
132	dockerfile_info: DockerfileInfo,
133	should_suggest_renames: bool = True,
134	) -> DockerBuildContext:
135	interpolation_context: dict[str, dict[str, str] \| InterpolationValue] = {}	×
136
137	if build_args:	×
138	interpolation_context["build_args"] = cls._merge_build_args(	×
139	dockerfile_info, build_args, build_env
140	)
141
142	# Override default value type for the `build_args` context to get helpful error messages.
143	interpolation_context["build_args"] = DockerBuildArgsInterpolationValue(	×
144	interpolation_context.get("build_args", {})
145	)
146
147	# Data from Pants.
148	interpolation_context["pants"] = {	×
149	# Present hash for all inputs that can be used for image tagging.
150	"hash": stable_hash((build_args, build_env, snapshot.digest)),
151	}
152
153	# Base image tags values for all stages (as parsed from the Dockerfile instructions).
154	stage_names, tags_values = cls._get_stages_and_tags(	×
155	dockerfile_info, interpolation_context["build_args"]
156	)
157	interpolation_context["tags"] = tags_values	×
158
159	copy_source_vs_context_source = (	×
160	tuple(
161	suggest_renames(
162	tentative_paths=(
163	# We don't want to include the Dockerfile as a suggested rename
164	dockerfile_info.source,
165	*dockerfile_info.copy_source_paths,
166	),
167	actual_files=snapshot.files,
168	actual_dirs=snapshot.dirs,
169	)
170	)
171	if should_suggest_renames
172	else ()
173	)
174
175	return cls(	×
176	build_args=build_args,
177	digest=snapshot.digest,
178	dockerfile=dockerfile_info.source,
179	build_env=build_env,
180	upstream_image_ids=tuple(sorted(upstream_image_ids)),
181	interpolation_context=InterpolationContext.from_dict(interpolation_context),
182	copy_source_vs_context_source=copy_source_vs_context_source,
183	stages=tuple(sorted(stage_names)),
184	)
185
186	@classmethod	5✔
187	def _get_stages_and_tags(	5✔
188	cls, dockerfile_info: DockerfileInfo, build_args: Mapping[str, str]
189	) -> tuple[set[str], dict[str, str]]:
190	# Go over all FROM tags and names for all stages.
191	stage_names: set[str] = set()	×
192	# tag is empty if image is referenced by digest instead
193	stage_tags = ([*tag.split(maxsplit=1), ""][:2] for tag in dockerfile_info.version_tags)	×
194	tags_values: dict[str, str] = {}	×
195	for idx, (stage, tag) in enumerate(stage_tags):	×
196	if tag.startswith("build-arg:"):	×
197	build_arg = tag[10:]	×
198	image_ref = build_args.get(build_arg) or dockerfile_info.build_args.to_dict().get(	×
199	build_arg
200	)
201	if not image_ref:	×
202	raise DockerBuildContextError(	×
203	f"Failed to parse Dockerfile baseimage tag for stage {stage} in "
204	f"{dockerfile_info.address} target, unknown build ARG: {build_arg!r}."
205	)
206	parsed = re.match(image_ref_regexp, image_ref.strip("\"'"))	×
207	tag = parsed.group("tag") or (parsed.group("digest") and "latest") if parsed else ""	×
208	if not tag:	×
209	raise DockerBuildContextError(	×
210	f"Failed to parse Dockerfile baseimage tag for stage {stage} in "
211	f"{dockerfile_info.address} target, from image ref: {image_ref}."
212	)
213
214	if stage != f"stage{idx}":	×
215	stage_names.add(stage)	×
216	if tag:	×
217	if idx == 0:	×
218	# Expose the first (stage0) FROM directive as the "baseimage".
219	tags_values["baseimage"] = tag	×
220	tags_values[stage] = tag	×
221
222	return stage_names, tags_values	×
223
224	@staticmethod	5✔
225	def _merge_build_args(	5✔
226	dockerfile_info: DockerfileInfo,
227	build_args: DockerBuildArgs,
228	build_env: DockerBuildEnvironment,
229	) -> dict[str, str]:
230	# Extract default arg values from the parsed Dockerfile.
231	build_arg_defaults = {	×
232	def_name: def_value
233	for def_name, has_default, def_value in [
234	def_arg.partition("=") for def_arg in dockerfile_info.build_args
235	]
236	if has_default
237	}
238	try:	×
239	# Create build args context value, based on defined build_args and
240	# extra_build_args. We do _not_ auto "magically" pick up all ARG names from the
241	# Dockerfile as first class args to use as placeholders, to make it more explicit
242	# which args are actually being used by Pants. We do pick up any defined default ARG
243	# values from the Dockerfile however, in order to not having to duplicate them in
244	# the BUILD files.
245	return {	×
246	arg_name: (
247	arg_value
248	if has_value
249	else build_env.get(arg_name, build_arg_defaults.get(arg_name))
250	)
251	for arg_name, has_value, arg_value in [
252	build_arg.partition("=") for build_arg in build_args
253	]
254	}
255	except DockerBuildEnvironmentError as e:	×
256	raise DockerBuildContextError(	×
257	f"Undefined value for build arg on the {dockerfile_info.address} target: {e}"
258	"\n\nIf you did not intend to inherit the value for this build arg from the "
259	"environment, provide a default value with the option `[docker].build_args` "
260	"or in the `extra_build_args` field on the target definition. Alternatively, "
261	"you may also provide a default value on the `ARG` instruction directly in "
262	"the `Dockerfile`."
263	) from e
264
265
266	@rule	5✔
267	async def create_docker_build_context(	5✔
268	request: DockerBuildContextRequest,
269	options: DockerOptions,
270	) -> DockerBuildContext:
271	# Get all targets to include in context.
272	transitive_targets = await transitive_targets_get(	×
273	TransitiveTargetsRequest([request.address]), **implicitly()
274	)
275	docker_image = transitive_targets.roots[0]	×
276
277	# Get all dependencies for the root target.
278	root_dependencies = await resolve_targets(	×
279	**implicitly(DependenciesRequest(docker_image.get(Dependencies)))
280	)
281
282	# Get all file sources from the root dependencies. That includes any non-file sources that can
283	# be "codegen"ed into a file source.
284	sources_request = determine_source_files(	×
285	SourceFilesRequest(
286	sources_fields=[tgt.get(SourcesField) for tgt in root_dependencies],
287	for_sources_types=(
288	DockerContextFilesSourcesField,
289	FileSourceField,
290	),
291	enable_codegen=True,
292	)
293	)
294
295	embedded_pkgs_per_target_request = find_valid_field_sets(	×
296	FieldSetsPerTargetRequest(PackageFieldSet, transitive_targets.dependencies), **implicitly()
297	)
298
299	sources, embedded_pkgs_per_target, dockerfile_info = await concurrently(	×
300	sources_request,
301	embedded_pkgs_per_target_request,
302	parse_dockerfile(DockerfileInfoRequest(docker_image.address), **implicitly()),
303	)
304
305	# Package binary dependencies for build context.
306	pkgs_wanting_embedding = [	×
307	field_set
308	for field_set in embedded_pkgs_per_target.field_sets
309	# Exclude docker images, unless build_upstream_images is true.
310	if (
311	request.build_upstream_images
312	or not isinstance(getattr(field_set, "source", None), DockerImageSourceField)
313	)
314	]
315	embedded_pkgs = await concurrently(	×
316	environment_aware_package(EnvironmentAwarePackageRequest(field_set))
317	for field_set in pkgs_wanting_embedding
318	)
319
320	if request.build_upstream_images:	×
321	images_str = ", ".join(	×
322	a.tags[0] for p in embedded_pkgs for a in p.artifacts if isinstance(a, BuiltDockerImage)
323	)
324	if images_str:	×
325	logger.debug(f"Built upstream Docker images: {images_str}")	×
326	else:
327	logger.debug("Did not build any upstream Docker images")	×
328
329	packages_str = ", ".join(a.relpath for p in embedded_pkgs for a in p.artifacts if a.relpath)	×
330	if packages_str:	×
331	logger.debug(f"Built packages for Docker image: {packages_str}")	×
332	else:
333	logger.debug("Did not build any packages for Docker image")	×
334
335	embedded_pkgs_digest = [built_package.digest for built_package in embedded_pkgs]	×
336	all_digests = (dockerfile_info.digest, sources.snapshot.digest, *embedded_pkgs_digest)	×
337
338	# Merge all digests to get the final docker build context digest.
339	context_request = digest_to_snapshot(**implicitly(MergeDigests(d for d in all_digests if d)))	×
340
341	# Requests for build args and env
342	build_args_request = docker_build_args(DockerBuildArgsRequest(docker_image), **implicitly())	×
343	build_env_request = docker_build_environment_vars(	×
344	DockerBuildEnvironmentRequest(docker_image), **implicitly()
345	)
346	context, supplied_build_args, build_env = await concurrently(	×
347	context_request, build_args_request, build_env_request
348	)
349
350	build_args = supplied_build_args	×
351
352	upstream_image_ids = []	×
353	if request.build_upstream_images:	×
354	# Update build arg values for FROM image build args.
355
356	# Get the FROM image build args with defined values in the Dockerfile & build args.
357	dockerfile_build_args = dockerfile_info.from_image_build_args.with_overrides(	×
358	supplied_build_args
359	).nonempty()
360	# Parse the build args values into Address instances.
361	from_image_addresses = await resolve_unparsed_address_inputs(	×
362	UnparsedAddressInputs(
363	dockerfile_build_args.values(),
364	owning_address=dockerfile_info.address,
365	description_of_origin=softwrap(
366	f"""
367	the FROM arguments from the file {dockerfile_info.source}
368	from the target {dockerfile_info.address}
369	"""
370	),
371	skip_invalid_addresses=True,
372	),
373	**implicitly(),
374	)
375	# Map those addresses to the corresponding built image ref (tag).
376	address_to_built_image_tag = {	×
377	field_set.address: image.tags[0]
378	for field_set, built in zip(embedded_pkgs_per_target.field_sets, embedded_pkgs)
379	for image in built.artifacts
380	if isinstance(image, BuiltDockerImage)
381	}
382	upstream_image_ids = [	×
383	image.image_id
384	for built in embedded_pkgs
385	for image in built.artifacts
386	if isinstance(image, BuiltDockerImage)
387	]
388	# Create the FROM image build args.
389	from_image_build_args = [	×
390	f"{arg_name}={address_to_built_image_tag[addr]}"
391	for arg_name, addr in zip(dockerfile_build_args.keys(), from_image_addresses)
392	]
393	build_args = build_args.extended(from_image_build_args)	×
394
395	# Render build args for turning COPY values in ARGS which are targets into their output
396	dockerfile_copy_args = dockerfile_info.copy_build_args.with_overrides(	×
397	supplied_build_args
398	).nonempty()
399
400	def get_artifact_paths(built_package: BuiltPackage) -> list[str]:	×
401	return [e.relpath for e in built_package.artifacts if e.relpath]	×
402
403	addrs_to_paths = {	×
404	field_set.address: get_artifact_paths(pkg)
405	for field_set, pkg in zip(embedded_pkgs_per_target.field_sets, embedded_pkgs)
406	}
407
408	copy_arg_as_build_args = await fill_args_from_copy(	×
409	dockerfile_copy_args, dockerfile_info, addrs_to_paths
410	)
411
412	build_args = build_args.extended(copy_arg_as_build_args)	×
413
414	return DockerBuildContext.create(	×
415	build_args=build_args,
416	snapshot=context,
417	upstream_image_ids=upstream_image_ids,
418	dockerfile_info=dockerfile_info,
419	build_env=build_env,
420	should_suggest_renames=options.suggest_renames,
421	)
422
423
424	async def fill_args_from_copy(	5✔
425	dockerfile_copy_args: dict[str, str], dockerfile_info, addrs_to_paths
426	):
427	copy_arg_addresses = await resolve_unparsed_address_inputs(	×
428	UnparsedAddressInputs(
429	dockerfile_info.copy_build_args.to_dict().values(),
430	owning_address=dockerfile_info.address,
431	description_of_origin=softwrap(
432	f"""
433	the COPY arguments from the file {dockerfile_info.source}
434	from the target {dockerfile_info.address}
435	"""
436	),
437	skip_invalid_addresses=True,
438	),
439	**implicitly(),
440	)
441
442	def resolve_arg(arg_name, maybe_addr) -> str:	×
443	if maybe_addr in addrs_to_paths:	×
444	return f"{arg_name}={shlex.join(addrs_to_paths[maybe_addr])}"	×
445	else:
446	# When the ARG value is a reference to a normal file
447	return f"{arg_name}={maybe_addr}"	×
448
449	copy_arg_as_build_args = [	×
450	resolve_arg(arg_name, arg_value)
451	for arg_name, arg_value in (zip(dockerfile_copy_args.keys(), copy_arg_addresses))
452	]
453	return copy_arg_as_build_args	×
454
455
456	def rules():	5✔
457	return (	5✔
458	*collect_rules(),
459	UnionRule(GenerateSourcesRequest, GenerateDockerContextFiles),
460	)

pantsbuild / pants / 18517631058

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous