21726110101

Committed 05 Feb 2026 07:49PM UTC coverage: 80.284% (-0.01%) from 80.296%

Build # 21726110101

Build Type

push

github

Committed by

web-flow

Commit Message

Support dep inference on multiple files in a single workunit (#23075)

Each file's result is still cached independently.

In large repos we schedule a large number of very fast
tasks, so the scheduling overhead dominates the runtime,
so processing many files in a batch is a win.

The Python-side code doesn't make use of this yet, so it
passes single files and receives singletons back.

A followup change can actually put this into action.

Run Details

20 of 44 new or added lines in 5 files covered. (45.45%)

2 existing lines in 2 files now uncovered.

78530 of 97815 relevant lines covered (80.28%)

3.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

65.0

/src/python/pants/backend/docker/subsystems/dockerfile_parser.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import json
from dataclasses import dataclass
from pathlib import PurePath

from pants.backend.docker.target_types import DockerImageSourceField
from pants.backend.docker.util_rules.docker_build_args import DockerBuildArgs
from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase
from pants.backend.python.target_types import EntryPoint
from pants.backend.python.util_rules import pex
from pants.backend.python.util_rules.pex import (
    VenvPex,
    VenvPexProcess,
    create_venv_pex,
    setup_venv_pex_process,
)
from pants.base.deprecated import warn_or_error
from pants.engine.addresses import Address
from pants.engine.fs import CreateDigest, Digest, FileContent
from pants.engine.internals.graph import hydrate_sources, resolve_target
from pants.engine.internals.native_engine import NativeDependenciesRequest
from pants.engine.intrinsics import create_digest, parse_dockerfile_info
from pants.engine.process import Process, execute_process_or_raise
from pants.engine.rules import collect_rules, implicitly, rule
from pants.engine.target import HydrateSourcesRequest, SourcesField, WrappedTargetRequest
from pants.option.option_types import BoolOption
from pants.util.docutil import bin_name, doc_url
from pants.util.logging import LogLevel
from pants.util.resources import read_resource
from pants.util.strutil import softwrap

_DOCKERFILE_SANDBOX_TOOL = "dockerfile_wrapper_script.py"
_DOCKERFILE_PACKAGE = "pants.backend.docker.subsystems"


class DockerfileParser(PythonToolRequirementsBase):
    options_scope = "dockerfile-parser"
    help_short = "Used to parse Dockerfile build specs to infer their dependencies."

    default_requirements = ["dockerfile>=3.2.0,<4"]

    register_interpreter_constraints = True

    default_lockfile_resource = (_DOCKERFILE_PACKAGE, "dockerfile.lock")

    use_rust_parser = BoolOption(
        default=True,
        help=softwrap(
            f"""
            Use the new Rust-based, multithreaded, in-process dependency parser.

            This new parser does not require the `dockerfile` dependency and thus, for instance,
            doesn't require Go to be installed to run on platforms for which that package doesn't
            provide pre-built wheels.

            If you think the new behaviour is causing problems, it is recommended that you run
            `{bin_name()} --dockerfile-parser-use-rust-parser=True peek :: > new-parser.json` and then
            `{bin_name()} --dockerfile-parser-use-rust-parser=False peek :: > old-parser.json` and compare the
            two results.

            If you think there is a bug, please file an issue:
            https://github.com/pantsbuild/pants/issues/new/choose.
            """
        ),
    )


@dataclass(frozen=True)
class ParserSetup:
    pex: VenvPex


@rule
async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup:
    parser_script_content = read_resource(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL)
    if not parser_script_content:
        raise ValueError(
            f"Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in {_DOCKERFILE_PACKAGE}."
        )

    parser_content = FileContent(
        path="__pants_df_parser.py",
        content=parser_script_content,
        is_executable=True,
    )
    parser_digest = await create_digest(CreateDigest([parser_content]))

    parser_pex = await create_venv_pex(
        **implicitly(
            dockerfile_parser.to_pex_request(
                main=EntryPoint(PurePath(parser_content.path).stem), sources=parser_digest
            )
        )
    )
    return ParserSetup(parser_pex)


@dataclass(frozen=True)
class DockerfileParseRequest:
    sources_digest: Digest
    args: tuple[str, ...]


@rule
async def setup_process_for_parse_dockerfile(
    request: DockerfileParseRequest, parser: ParserSetup
) -> Process:
    process = await setup_venv_pex_process(
        VenvPexProcess(
            parser.pex,
            argv=request.args,
            description="Parse Dockerfile.",
            input_digest=request.sources_digest,
            level=LogLevel.DEBUG,
        ),
        **implicitly(),
    )
    return process


class DockerfileInfoError(Exception):
    pass


@dataclass(frozen=True)
class DockerfileInfo:
    address: Address
    digest: Digest

    # Data from the parsed Dockerfile, keep in sync with
    # `dockerfile_wrapper_script.py:ParsedDockerfileInfo`:
    source: str
    build_args: DockerBuildArgs = DockerBuildArgs()
    copy_source_paths: tuple[str, ...] = ()
    copy_build_args: DockerBuildArgs = DockerBuildArgs()
    from_image_build_args: DockerBuildArgs = DockerBuildArgs()
    version_tags: tuple[str, ...] = ()


@dataclass(frozen=True)
class DockerfileInfoRequest:
    address: Address


async def _natively_parse_dockerfile(address: Address, digest: Digest) -> DockerfileInfo:
    results = await parse_dockerfile_info(NativeDependenciesRequest(digest))
    assert len(results.path_to_infos) == 1
    result = next(iter(results.path_to_infos.values()))
    return DockerfileInfo(
        address=address,
        digest=digest,
        source=result.source,
        build_args=DockerBuildArgs.from_strings(*result.build_args, duplicates_must_match=True),
        copy_source_paths=tuple(result.copy_source_paths),
        copy_build_args=DockerBuildArgs.from_strings(
            *result.copy_build_args, duplicates_must_match=True
        ),
        from_image_build_args=DockerBuildArgs.from_strings(
            *result.from_image_build_args, duplicates_must_match=True
        ),
        version_tags=tuple(result.version_tags),
    )


async def _legacy_parse_dockerfile(
    address: Address, digest: Digest, dockerfiles: tuple[str, ...]
) -> DockerfileInfo:
    result = await execute_process_or_raise(
        **implicitly(DockerfileParseRequest(digest, dockerfiles))
    )

    try:
        raw_output = result.stdout.decode("utf-8")
        outputs = json.loads(raw_output)
        assert len(outputs) == len(dockerfiles)
    except Exception as e:
        raise DockerfileInfoError(
            f"Unexpected failure to parse Dockerfiles: {', '.join(dockerfiles)}, "
            f"for the {address} target: {e}\nDockerfile parser output:\n{raw_output}"
        ) from e
    info = outputs[0]
    return DockerfileInfo(
        address=address,
        digest=digest,
        source=info["source"],
        build_args=DockerBuildArgs.from_strings(*info["build_args"], duplicates_must_match=True),
        copy_source_paths=tuple(info["copy_source_paths"]),
        copy_build_args=DockerBuildArgs.from_strings(
            *info["copy_build_args"], duplicates_must_match=True
        ),
        from_image_build_args=DockerBuildArgs.from_strings(
            *info["from_image_build_args"], duplicates_must_match=True
        ),
        version_tags=tuple(info["version_tags"]),
    )


@rule
async def parse_dockerfile(
    request: DockerfileInfoRequest, dockerfile_parser: DockerfileParser
) -> DockerfileInfo:
    wrapped_target = await resolve_target(
        WrappedTargetRequest(request.address, description_of_origin="<infallible>"), **implicitly()
    )
    target = wrapped_target.target
    sources = await hydrate_sources(
        HydrateSourcesRequest(
            target.get(SourcesField),
            for_sources_types=(DockerImageSourceField,),
            enable_codegen=True,
        ),
        **implicitly(),
    )

    dockerfiles = sources.snapshot.files
    assert len(dockerfiles) == 1, (
        f"Internal error: Expected a single source file to Dockerfile parse request {request}, "
        f"got: {dockerfiles}."
    )

    if not dockerfile_parser.use_rust_parser:
        warn_or_error(
            removal_version="2.32.0.dev1",
            entity="Using the old Dockerfile parser",
            hint=softwrap(
                f"""
                Future versions of Pants will only support the Rust-based parser for Dockerfiles. The new
                parser is faster and does not require installing extra dependencies.

                The `[dockerfile-parser].use_rust_parser` option is currently explicitly set to `false` to
                force the use of the old parser. This parser will be removed in future.

                Please remove this setting to use the new parser. If you find issues with the new parser,
                please let us know: <https://github.com/pantsbuild/pants/issues/new/choose>

                See {doc_url("reference/subsystems/dockerfile-parser#use_rust_parser")} for
                additional information.
                """
            ),
        )

    try:
        if dockerfile_parser.use_rust_parser:
            return await _natively_parse_dockerfile(target.address, sources.snapshot.digest)
        else:
            return await _legacy_parse_dockerfile(
                target.address, sources.snapshot.digest, dockerfiles
            )
    except ValueError as e:
        raise DockerfileInfoError(
            f"Error while parsing {dockerfiles[0]} for the {request.address} target: {e}"
        ) from e


def rules():
    return (
        *collect_rules(),
        *pex.rules(),
    )

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	9✔
5
6	import json	9✔
7	from dataclasses import dataclass	9✔
8	from pathlib import PurePath	9✔
9
10	from pants.backend.docker.target_types import DockerImageSourceField	9✔
11	from pants.backend.docker.util_rules.docker_build_args import DockerBuildArgs	9✔
12	from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase	9✔
13	from pants.backend.python.target_types import EntryPoint	9✔
14	from pants.backend.python.util_rules import pex	9✔
15	from pants.backend.python.util_rules.pex import (	9✔
16	VenvPex,
17	VenvPexProcess,
18	create_venv_pex,
19	setup_venv_pex_process,
20	)
21	from pants.base.deprecated import warn_or_error	9✔
22	from pants.engine.addresses import Address	9✔
23	from pants.engine.fs import CreateDigest, Digest, FileContent	9✔
24	from pants.engine.internals.graph import hydrate_sources, resolve_target	9✔
25	from pants.engine.internals.native_engine import NativeDependenciesRequest	9✔
26	from pants.engine.intrinsics import create_digest, parse_dockerfile_info	9✔
27	from pants.engine.process import Process, execute_process_or_raise	9✔
28	from pants.engine.rules import collect_rules, implicitly, rule	9✔
29	from pants.engine.target import HydrateSourcesRequest, SourcesField, WrappedTargetRequest	9✔
30	from pants.option.option_types import BoolOption	9✔
31	from pants.util.docutil import bin_name, doc_url	9✔
32	from pants.util.logging import LogLevel	9✔
33	from pants.util.resources import read_resource	9✔
34	from pants.util.strutil import softwrap	9✔
35
36	_DOCKERFILE_SANDBOX_TOOL = "dockerfile_wrapper_script.py"	9✔
37	_DOCKERFILE_PACKAGE = "pants.backend.docker.subsystems"	9✔
38
39
40	class DockerfileParser(PythonToolRequirementsBase):	9✔
41	options_scope = "dockerfile-parser"	9✔
42	help_short = "Used to parse Dockerfile build specs to infer their dependencies."	9✔
43
44	default_requirements = ["dockerfile>=3.2.0,<4"]	9✔
45
46	register_interpreter_constraints = True	9✔
47
48	default_lockfile_resource = (_DOCKERFILE_PACKAGE, "dockerfile.lock")	9✔
49
50	use_rust_parser = BoolOption(	9✔
51	default=True,
52	help=softwrap(
53	f"""
54	Use the new Rust-based, multithreaded, in-process dependency parser.
55
56	This new parser does not require the `dockerfile` dependency and thus, for instance,
57	doesn't require Go to be installed to run on platforms for which that package doesn't
58	provide pre-built wheels.
59
60	If you think the new behaviour is causing problems, it is recommended that you run
61	`{bin_name()} --dockerfile-parser-use-rust-parser=True peek :: > new-parser.json` and then
62	`{bin_name()} --dockerfile-parser-use-rust-parser=False peek :: > old-parser.json` and compare the
63	two results.
64
65	If you think there is a bug, please file an issue:
66	https://github.com/pantsbuild/pants/issues/new/choose.
67	"""
68	),
69	)
70
71
72	@dataclass(frozen=True)	9✔
73	class ParserSetup:	9✔
74	pex: VenvPex	9✔
75
76
77	@rule	9✔
78	async def setup_parser(dockerfile_parser: DockerfileParser) -> ParserSetup:	9✔
79	parser_script_content = read_resource(_DOCKERFILE_PACKAGE, _DOCKERFILE_SANDBOX_TOOL)	×
80	if not parser_script_content:	×
81	raise ValueError(	×
82	f"Unable to find source to {_DOCKERFILE_SANDBOX_TOOL!r} in {_DOCKERFILE_PACKAGE}."
83	)
84
85	parser_content = FileContent(	×
86	path="__pants_df_parser.py",
87	content=parser_script_content,
88	is_executable=True,
89	)
90	parser_digest = await create_digest(CreateDigest([parser_content]))	×
91
92	parser_pex = await create_venv_pex(	×
93	**implicitly(
94	dockerfile_parser.to_pex_request(
95	main=EntryPoint(PurePath(parser_content.path).stem), sources=parser_digest
96	)
97	)
98	)
99	return ParserSetup(parser_pex)	×
100
101
102	@dataclass(frozen=True)	9✔
103	class DockerfileParseRequest:	9✔
104	sources_digest: Digest	9✔
105	args: tuple[str, ...]	9✔
106
107
108	@rule	9✔
109	async def setup_process_for_parse_dockerfile(	9✔
110	request: DockerfileParseRequest, parser: ParserSetup
111	) -> Process:
112	process = await setup_venv_pex_process(	×
113	VenvPexProcess(
114	parser.pex,
115	argv=request.args,
116	description="Parse Dockerfile.",
117	input_digest=request.sources_digest,
118	level=LogLevel.DEBUG,
119	),
120	**implicitly(),
121	)
122	return process	×
123
124
125	class DockerfileInfoError(Exception):	9✔
126	pass	9✔
127
128
129	@dataclass(frozen=True)	9✔
130	class DockerfileInfo:	9✔
131	address: Address	9✔
132	digest: Digest	9✔
133
134	# Data from the parsed Dockerfile, keep in sync with
135	# `dockerfile_wrapper_script.py:ParsedDockerfileInfo`:
136	source: str	9✔
137	build_args: DockerBuildArgs = DockerBuildArgs()	9✔
138	copy_source_paths: tuple[str, ...] = ()	9✔
139	copy_build_args: DockerBuildArgs = DockerBuildArgs()	9✔
140	from_image_build_args: DockerBuildArgs = DockerBuildArgs()	9✔
141	version_tags: tuple[str, ...] = ()	9✔
142
143
144	@dataclass(frozen=True)	9✔
145	class DockerfileInfoRequest:	9✔
146	address: Address	9✔
147
148
149	async def _natively_parse_dockerfile(address: Address, digest: Digest) -> DockerfileInfo:	9✔
NEW 150	results = await parse_dockerfile_info(NativeDependenciesRequest(digest))	×
NEW 151	assert len(results.path_to_infos) == 1	×
NEW 152	result = next(iter(results.path_to_infos.values()))	×
UNCOV 153	return DockerfileInfo(	×
154	address=address,
155	digest=digest,
156	source=result.source,
157	build_args=DockerBuildArgs.from_strings(*result.build_args, duplicates_must_match=True),
158	copy_source_paths=tuple(result.copy_source_paths),
159	copy_build_args=DockerBuildArgs.from_strings(
160	*result.copy_build_args, duplicates_must_match=True
161	),
162	from_image_build_args=DockerBuildArgs.from_strings(
163	*result.from_image_build_args, duplicates_must_match=True
164	),
165	version_tags=tuple(result.version_tags),
166	)
167
168
169	async def _legacy_parse_dockerfile(	9✔
170	address: Address, digest: Digest, dockerfiles: tuple[str, ...]
171	) -> DockerfileInfo:
172	result = await execute_process_or_raise(	×
173	**implicitly(DockerfileParseRequest(digest, dockerfiles))
174	)
175
176	try:	×
177	raw_output = result.stdout.decode("utf-8")	×
178	outputs = json.loads(raw_output)	×
179	assert len(outputs) == len(dockerfiles)	×
180	except Exception as e:	×
181	raise DockerfileInfoError(	×
182	f"Unexpected failure to parse Dockerfiles: {', '.join(dockerfiles)}, "
183	f"for the {address} target: {e}\nDockerfile parser output:\n{raw_output}"
184	) from e
185	info = outputs[0]	×
186	return DockerfileInfo(	×
187	address=address,
188	digest=digest,
189	source=info["source"],
190	build_args=DockerBuildArgs.from_strings(*info["build_args"], duplicates_must_match=True),
191	copy_source_paths=tuple(info["copy_source_paths"]),
192	copy_build_args=DockerBuildArgs.from_strings(
193	*info["copy_build_args"], duplicates_must_match=True
194	),
195	from_image_build_args=DockerBuildArgs.from_strings(
196	*info["from_image_build_args"], duplicates_must_match=True
197	),
198	version_tags=tuple(info["version_tags"]),
199	)
200
201
202	@rule	9✔
203	async def parse_dockerfile(	9✔
204	request: DockerfileInfoRequest, dockerfile_parser: DockerfileParser
205	) -> DockerfileInfo:
206	wrapped_target = await resolve_target(	×
207	WrappedTargetRequest(request.address, description_of_origin="<infallible>"), **implicitly()
208	)
209	target = wrapped_target.target	×
210	sources = await hydrate_sources(	×
211	HydrateSourcesRequest(
212	target.get(SourcesField),
213	for_sources_types=(DockerImageSourceField,),
214	enable_codegen=True,
215	),
216	**implicitly(),
217	)
218
219	dockerfiles = sources.snapshot.files	×
220	assert len(dockerfiles) == 1, (	×
221	f"Internal error: Expected a single source file to Dockerfile parse request {request}, "
222	f"got: {dockerfiles}."
223	)
224
225	if not dockerfile_parser.use_rust_parser:	×
226	warn_or_error(	×
227	removal_version="2.32.0.dev1",
228	entity="Using the old Dockerfile parser",
229	hint=softwrap(
230	f"""
231	Future versions of Pants will only support the Rust-based parser for Dockerfiles. The new
232	parser is faster and does not require installing extra dependencies.
233
234	The `[dockerfile-parser].use_rust_parser` option is currently explicitly set to `false` to
235	force the use of the old parser. This parser will be removed in future.
236
237	Please remove this setting to use the new parser. If you find issues with the new parser,
238	please let us know: <https://github.com/pantsbuild/pants/issues/new/choose>
239
240	See {doc_url("reference/subsystems/dockerfile-parser#use_rust_parser")} for
241	additional information.
242	"""
243	),
244	)
245
246	try:	×
247	if dockerfile_parser.use_rust_parser:	×
248	return await _natively_parse_dockerfile(target.address, sources.snapshot.digest)	×
249	else:
250	return await _legacy_parse_dockerfile(	×
251	target.address, sources.snapshot.digest, dockerfiles
252	)
253	except ValueError as e:	×
254	raise DockerfileInfoError(	×
255	f"Error while parsing {dockerfiles[0]} for the {request.address} target: {e}"
256	) from e
257
258
259	def rules():	9✔
260	return (	9✔
261	*collect_rules(),
262	*pex.rules(),
263	)

pantsbuild / pants / 21726110101

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous