24055979590

Committed 06 Apr 2026 11:17PM UTC coverage: 52.37% (-40.5%) from 92.908%

Build # 24055979590

Build Type

Pull #23225

github

Committed by

web-flow

Commit Message

Merge 67474653c into 542ca048d

Pull Request Pull Request #23225: Add --test-show-all-batch-targets to expose all targets in batched pytest

Coverage Stats

6 of 17 new or added lines in 2 files covered. (35.29%)

23030 existing lines in 605 files now uncovered.

31643 of 60422 relevant lines covered (52.37%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.0

/src/python/pants/backend/docker/subsystems/dockerfile_parser.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import json
from dataclasses import dataclass
from pathlib import PurePath

from pants.backend.docker.target_types import DockerImageSourceField
from pants.backend.docker.util_rules.docker_build_args import DockerBuildArgs
from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase
from pants.backend.python.target_types import EntryPoint
from pants.backend.python.util_rules import pex
from pants.backend.python.util_rules.pex import (
    VenvPex,
    VenvPexProcess,
    create_venv_pex,
    setup_venv_pex_process,
)
from pants.base.deprecated import warn_or_error
from pants.engine.addresses import Address
from pants.engine.fs import CreateDigest, Digest, FileContent
from pants.engine.internals.graph import hydrate_sources, resolve_target
from pants.engine.internals.native_engine import NativeDependenciesRequest
from pants.engine.intrinsics import create_digest, parse_dockerfile_info
from pants.engine.process import Process, execute_process_or_raise
from pants.engine.rules import collect_rules, implicitly, rule
from pants.engine.target import HydrateSourcesRequest, SourcesField, WrappedTargetRequest
from pants.option.option_types import BoolOption
from pants.util.docutil import bin_name, doc_url
from pants.util.logging import LogLevel
from pants.util.resources import read_resource
from pants.util.strutil import softwrap

# pants: infer-dep(dockerfile.lock*)

_DOCKERFILE_SANDBOX_TOOL = "dockerfile_wrapper_script.py"
_DOCKERFILE_PACKAGE = "pants.backend.docker.subsystems"


class DockerfileParser(PythonToolRequirementsBase):
    options_scope = "dockerfile-parser"
    help_short = "Used to parse Dockerfile build specs to infer their dependencies."

    default_requirements = ["dockerfile>=3.2.0,<4"]

    register_interpreter_constraints = True

    default_lockfile_resource = (_DOCKERFILE_PACKAGE, "dockerfile.lock")

    use_rust_parser = BoolOption(
        default=True,
        help=softwrap(
            f"""
            Use the new Rust-based, multithreaded, in-process dependency parser.

            This new parser does not require the `dockerfile` dependency and thus, for instance,
            doesn't require Go to be installed to run on platforms for which that package doesn't
            provide pre-built wheels.

            If you think the new behaviour is causing problems, it is recommended that you run
            `{bin_name()} --dockerfile-parser-use-rust-parser=True peek :: > new-parser.json` and then
            `{bin_name()} --dockerfile-parser-use-rust-parser=False peek :: > old-parser.json` and compare the
            two results.

            If you think there is a bug, please file an issue:
            https://github.com/pantsbuild/pants/issues/new/choose.
            """
        ),
    )


@dataclass(frozen=True)
class ParserSetup:
    pex: VenvPex


@rule
async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup:
    parser_script_content = read_resource(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL)
    if not parser_script_content:
        raise ValueError(
            f"Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in {_DOCKERFILE_PACKAGE}."
        )

    parser_content = FileContent(
        path="__pants_df_parser.py",
        content=parser_script_content,
        is_executable=True,
    )
    parser_digest = await create_digest(CreateDigest([parser_content]))

    parser_pex = await create_venv_pex(
        **implicitly(
            dockerfile_parser.to_pex_request(
                main=EntryPoint(PurePath(parser_content.path).stem), sources=parser_digest
            )
        )
    )
    return ParserSetup(parser_pex)


@dataclass(frozen=True)
class DockerfileParseRequest:
    sources_digest: Digest
    args: tuple[str, ...]


@rule
async def setup_process_for_parse_dockerfile(
    request: DockerfileParseRequest, parser: ParserSetup
) -> Process:
    process = await setup_venv_pex_process(
        VenvPexProcess(
            parser.pex,
            argv=request.args,
            description="Parse Dockerfile.",
            input_digest=request.sources_digest,
            level=LogLevel.DEBUG,
        ),
        **implicitly(),
    )
    return process


class DockerfileInfoError(Exception):
    pass


@dataclass(frozen=True)
class DockerfileInfo:
    address: Address
    digest: Digest

    # Data from the parsed Dockerfile, keep in sync with
    # `dockerfile_wrapper_script.py:ParsedDockerfileInfo`:
    source: str
    build_args: DockerBuildArgs = DockerBuildArgs()
    copy_source_paths: tuple[str, ...] = ()
    copy_build_args: DockerBuildArgs = DockerBuildArgs()
    from_image_build_args: DockerBuildArgs = DockerBuildArgs()
    version_tags: tuple[str, ...] = ()


@dataclass(frozen=True)
class DockerfileInfoRequest:
    address: Address


async def _natively_parse_dockerfile(address: Address, digest: Digest) -> DockerfileInfo:
    results = await parse_dockerfile_info(NativeDependenciesRequest(digest))
    assert len(results.path_to_infos) == 1
    result = next(iter(results.path_to_infos.values()))
    return DockerfileInfo(
        address=address,
        digest=digest,
        source=result.source,
        build_args=DockerBuildArgs.from_strings(*result.build_args, duplicates_must_match=True),
        copy_source_paths=tuple(result.copy_source_paths),
        copy_build_args=DockerBuildArgs.from_strings(
            *result.copy_build_args, duplicates_must_match=True
        ),
        from_image_build_args=DockerBuildArgs.from_strings(
            *result.from_image_build_args, duplicates_must_match=True
        ),
        version_tags=tuple(result.version_tags),
    )


async def _legacy_parse_dockerfile(
    address: Address, digest: Digest, dockerfiles: tuple[str, ...]
) -> DockerfileInfo:
    result = await execute_process_or_raise(
        **implicitly(DockerfileParseRequest(digest, dockerfiles))
    )

    try:
        raw_output = result.stdout.decode("utf-8")
        outputs = json.loads(raw_output)
        assert len(outputs) == len(dockerfiles)
    except Exception as e:
        raise DockerfileInfoError(
            f"Unexpected failure to parse Dockerfiles: {', '.join(dockerfiles)}, "
            f"for the {address} target: {e}\nDockerfile parser output:\n{raw_output}"
        ) from e
    info = outputs[0]
    return DockerfileInfo(
        address=address,
        digest=digest,
        source=info["source"],
        build_args=DockerBuildArgs.from_strings(*info["build_args"], duplicates_must_match=True),
        copy_source_paths=tuple(info["copy_source_paths"]),
        copy_build_args=DockerBuildArgs.from_strings(
            *info["copy_build_args"], duplicates_must_match=True
        ),
        from_image_build_args=DockerBuildArgs.from_strings(
            *info["from_image_build_args"], duplicates_must_match=True
        ),
        version_tags=tuple(info["version_tags"]),
    )


@rule
async def parse_dockerfile(
    request: DockerfileInfoRequest, dockerfile_parser: DockerfileParser
) -> DockerfileInfo:
    wrapped_target = await resolve_target(
        WrappedTargetRequest(request.address, description_of_origin="<infallible>"), **implicitly()
    )
    target = wrapped_target.target
    sources = await hydrate_sources(
        HydrateSourcesRequest(
            target.get(SourcesField),
            for_sources_types=(DockerImageSourceField,),
            enable_codegen=True,
        ),
        **implicitly(),
    )

    dockerfiles = sources.snapshot.files
    assert len(dockerfiles) == 1, (
        f"Internal error: Expected a single source file to Dockerfile parse request {request}, "
        f"got: {dockerfiles}."
    )

    if not dockerfile_parser.use_rust_parser:
        warn_or_error(
            removal_version="2.33.0.dev1",
            entity="Using the old Dockerfile parser",
            hint=softwrap(
                f"""
                Future versions of Pants will only support the Rust-based parser for Dockerfiles. The new
                parser is faster and does not require installing extra dependencies.

                The `[dockerfile-parser].use_rust_parser` option is currently explicitly set to `false` to
                force the use of the old parser. This parser will be removed in future.

                Please remove this setting to use the new parser. If you find issues with the new parser,
                please let us know: <https://github.com/pantsbuild/pants/issues/new/choose>

                See {doc_url("reference/subsystems/dockerfile-parser#use_rust_parser")} for
                additional information.
                """
            ),
        )

    try:
        if dockerfile_parser.use_rust_parser:
            return await _natively_parse_dockerfile(target.address, sources.snapshot.digest)
        else:
            return await _legacy_parse_dockerfile(
                target.address, sources.snapshot.digest, dockerfiles
            )
    except ValueError as e:
        raise DockerfileInfoError(
            f"Error while parsing {dockerfiles[0]} for the {request.address} target: {e}"
        ) from e


def rules():
    return (
        *collect_rules(),
        *pex.rules(),
    )

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	2✔
5
6	import json	2✔
7	from dataclasses import dataclass	2✔
8	from pathlib import PurePath	2✔
9
10	from pants.backend.docker.target_types import DockerImageSourceField	2✔
11	from pants.backend.docker.util_rules.docker_build_args import DockerBuildArgs	2✔
12	from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase	2✔
13	from pants.backend.python.target_types import EntryPoint	2✔
14	from pants.backend.python.util_rules import pex	2✔
15	from pants.backend.python.util_rules.pex import (	2✔
16	VenvPex,
17	VenvPexProcess,
18	create_venv_pex,
19	setup_venv_pex_process,
20	)
21	from pants.base.deprecated import warn_or_error	2✔
22	from pants.engine.addresses import Address	2✔
23	from pants.engine.fs import CreateDigest, Digest, FileContent	2✔
24	from pants.engine.internals.graph import hydrate_sources, resolve_target	2✔
25	from pants.engine.internals.native_engine import NativeDependenciesRequest	2✔
26	from pants.engine.intrinsics import create_digest, parse_dockerfile_info	2✔
27	from pants.engine.process import Process, execute_process_or_raise	2✔
28	from pants.engine.rules import collect_rules, implicitly, rule	2✔
29	from pants.engine.target import HydrateSourcesRequest, SourcesField, WrappedTargetRequest	2✔
30	from pants.option.option_types import BoolOption	2✔
31	from pants.util.docutil import bin_name, doc_url	2✔
32	from pants.util.logging import LogLevel	2✔
33	from pants.util.resources import read_resource	2✔
34	from pants.util.strutil import softwrap	2✔
35
36	# pants: infer-dep(dockerfile.lock*)
37
38	_DOCKERFILE_SANDBOX_TOOL = "dockerfile_wrapper_script.py"	2✔
39	_DOCKERFILE_PACKAGE = "pants.backend.docker.subsystems"	2✔
40
41
42	class DockerfileParser(PythonToolRequirementsBase):	2✔
43	options_scope = "dockerfile-parser"	2✔
44	help_short = "Used to parse Dockerfile build specs to infer their dependencies."	2✔
45
46	default_requirements = ["dockerfile>=3.2.0,<4"]	2✔
47
48	register_interpreter_constraints = True	2✔
49
50	default_lockfile_resource = (_DOCKERFILE_PACKAGE, "dockerfile.lock")	2✔
51
52	use_rust_parser = BoolOption(	2✔
53	default=True,
54	help=softwrap(
55	f"""
56	Use the new Rust-based, multithreaded, in-process dependency parser.
57
58	This new parser does not require the `dockerfile` dependency and thus, for instance,
59	doesn't require Go to be installed to run on platforms for which that package doesn't
60	provide pre-built wheels.
61
62	If you think the new behaviour is causing problems, it is recommended that you run
63	`{bin_name()} --dockerfile-parser-use-rust-parser=True peek :: > new-parser.json` and then
64	`{bin_name()} --dockerfile-parser-use-rust-parser=False peek :: > old-parser.json` and compare the
65	two results.
66
67	If you think there is a bug, please file an issue:
68	https://github.com/pantsbuild/pants/issues/new/choose.
69	"""
70	),
71	)
72
73
74	@dataclass(frozen=True)	2✔
75	class ParserSetup:	2✔
76	pex: VenvPex	2✔
77
78
79	@rule	2✔
80	async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup:	2✔
UNCOV 81	parser_script_content = read_resource(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL)	×
UNCOV 82	if not parser_script_content:	×
83	raise ValueError(	×
84	f"Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in {_DOCKERFILE_PACKAGE}."
85	)
86
UNCOV 87	parser_content = FileContent(	×
88	path="__pants_df_parser.py",
89	content=parser_script_content,
90	is_executable=True,
91	)
UNCOV 92	parser_digest = await create_digest(CreateDigest([parser_content]))	×
93
UNCOV 94	parser_pex = await create_venv_pex(	×
95	**implicitly(
96	dockerfile_parser.to_pex_request(
97	main=EntryPoint(PurePath(parser_content.path).stem), sources=parser_digest
98	)
99	)
100	)
UNCOV 101	return ParserSetup(parser_pex)	×
102
103
104	@dataclass(frozen=True)	2✔
105	class DockerfileParseRequest:	2✔
106	sources_digest: Digest	2✔
107	args: tuple[str, ...]	2✔
108
109
110	@rule	2✔
111	async def setup_process_for_parse_dockerfile(	2✔
112	request: DockerfileParseRequest, parser: ParserSetup
113	) -> Process:
UNCOV 114	process = await setup_venv_pex_process(	×
115	VenvPexProcess(
116	parser.pex,
117	argv=request.args,
118	description="Parse Dockerfile.",
119	input_digest=request.sources_digest,
120	level=LogLevel.DEBUG,
121	),
122	**implicitly(),
123	)
UNCOV 124	return process	×
125
126
127	class DockerfileInfoError(Exception):	2✔
128	pass	2✔
129
130
131	@dataclass(frozen=True)	2✔
132	class DockerfileInfo:	2✔
133	address: Address	2✔
134	digest: Digest	2✔
135
136	# Data from the parsed Dockerfile, keep in sync with
137	# `dockerfile_wrapper_script.py:ParsedDockerfileInfo`:
138	source: str	2✔
139	build_args: DockerBuildArgs = DockerBuildArgs()	2✔
140	copy_source_paths: tuple[str, ...] = ()	2✔
141	copy_build_args: DockerBuildArgs = DockerBuildArgs()	2✔
142	from_image_build_args: DockerBuildArgs = DockerBuildArgs()	2✔
143	version_tags: tuple[str, ...] = ()	2✔
144
145
146	@dataclass(frozen=True)	2✔
147	class DockerfileInfoRequest:	2✔
148	address: Address	2✔
149
150
151	async def _natively_parse_dockerfile(address: Address, digest: Digest) -> DockerfileInfo:	2✔
152	results = await parse_dockerfile_info(NativeDependenciesRequest(digest))	2✔
153	assert len(results.path_to_infos) == 1	2✔
154	result = next(iter(results.path_to_infos.values()))	2✔
155	return DockerfileInfo(	2✔
156	address=address,
157	digest=digest,
158	source=result.source,
159	build_args=DockerBuildArgs.from_strings(*result.build_args, duplicates_must_match=True),
160	copy_source_paths=tuple(result.copy_source_paths),
161	copy_build_args=DockerBuildArgs.from_strings(
162	*result.copy_build_args, duplicates_must_match=True
163	),
164	from_image_build_args=DockerBuildArgs.from_strings(
165	*result.from_image_build_args, duplicates_must_match=True
166	),
167	version_tags=tuple(result.version_tags),
168	)
169
170
171	async def _legacy_parse_dockerfile(	2✔
172	address: Address, digest: Digest, dockerfiles: tuple[str, ...]
173	) -> DockerfileInfo:
UNCOV 174	result = await execute_process_or_raise(	×
175	**implicitly(DockerfileParseRequest(digest, dockerfiles))
176	)
177
UNCOV 178	try:	×
UNCOV 179	raw_output = result.stdout.decode("utf-8")	×
UNCOV 180	outputs = json.loads(raw_output)	×
UNCOV 181	assert len(outputs) == len(dockerfiles)	×
182	except Exception as e:	×
183	raise DockerfileInfoError(	×
184	f"Unexpected failure to parse Dockerfiles: {', '.join(dockerfiles)}, "
185	f"for the {address} target: {e}\nDockerfile parser output:\n{raw_output}"
186	) from e
UNCOV 187	info = outputs[0]	×
UNCOV 188	return DockerfileInfo(	×
189	address=address,
190	digest=digest,
191	source=info["source"],
192	build_args=DockerBuildArgs.from_strings(*info["build_args"], duplicates_must_match=True),
193	copy_source_paths=tuple(info["copy_source_paths"]),
194	copy_build_args=DockerBuildArgs.from_strings(
195	*info["copy_build_args"], duplicates_must_match=True
196	),
197	from_image_build_args=DockerBuildArgs.from_strings(
198	*info["from_image_build_args"], duplicates_must_match=True
199	),
200	version_tags=tuple(info["version_tags"]),
201	)
202
203
204	@rule	2✔
205	async def parse_dockerfile(	2✔
206	request: DockerfileInfoRequest, dockerfile_parser: DockerfileParser
207	) -> DockerfileInfo:
208	wrapped_target = await resolve_target(	2✔
209	WrappedTargetRequest(request.address, description_of_origin="<infallible>"), **implicitly()
210	)
211	target = wrapped_target.target	2✔
212	sources = await hydrate_sources(	2✔
213	HydrateSourcesRequest(
214	target.get(SourcesField),
215	for_sources_types=(DockerImageSourceField,),
216	enable_codegen=True,
217	),
218	**implicitly(),
219	)
220
221	dockerfiles = sources.snapshot.files	2✔
222	assert len(dockerfiles) == 1, (	2✔
223	f"Internal error: Expected a single source file to Dockerfile parse request {request}, "
224	f"got: {dockerfiles}."
225	)
226
227	if not dockerfile_parser.use_rust_parser:	2✔
UNCOV 228	warn_or_error(	×
229	removal_version="2.33.0.dev1",
230	entity="Using the old Dockerfile parser",
231	hint=softwrap(
232	f"""
233	Future versions of Pants will only support the Rust-based parser for Dockerfiles. The new
234	parser is faster and does not require installing extra dependencies.
235
236	The `[dockerfile-parser].use_rust_parser` option is currently explicitly set to `false` to
237	force the use of the old parser. This parser will be removed in future.
238
239	Please remove this setting to use the new parser. If you find issues with the new parser,
240	please let us know: <https://github.com/pantsbuild/pants/issues/new/choose>
241
242	See {doc_url("reference/subsystems/dockerfile-parser#use_rust_parser")} for
243	additional information.
244	"""
245	),
246	)
247
248	try:	2✔
249	if dockerfile_parser.use_rust_parser:	2✔
250	return await _natively_parse_dockerfile(target.address, sources.snapshot.digest)	2✔
251	else:
UNCOV 252	return await _legacy_parse_dockerfile(	×
253	target.address, sources.snapshot.digest, dockerfiles
254	)
UNCOV 255	except ValueError as e:	×
UNCOV 256	raise DockerfileInfoError(	×
257	f"Error while parsing {dockerfiles[0]} for the {request.address} target: {e}"
258	) from e
259
260
261	def rules():	2✔
262	return (	2✔
263	*collect_rules(),
264	*pex.rules(),
265	)

pantsbuild / pants / 24055979590

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous