26689585807

Committed 30 May 2026 04:55PM UTC coverage: 92.742% (-0.05%) from 92.792%

Build # 26689585807

Build Type

Pull #23343

github

Committed by

web-flow

Commit Message

Merge c42efe377 into c8127c1f4

Pull Request Pull Request #23343: Add buf as an alternate Python protobuf code generator

Coverage Stats

767 of 807 new or added lines in 17 files covered. (95.04%)

69 existing lines in 3 files now uncovered.

93753 of 101090 relevant lines covered (92.74%)

4.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.59

/src/python/pants/backend/codegen/protobuf/python/buf_rules.py

# Copyright 2026 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import logging
import os
from dataclasses import dataclass

from pants.backend.codegen.protobuf.buf.config import (
    MissingBufLockError,
    gen_template_request_for_target,
    parse_buf_yaml_deps,
    resolved_template_path,
    synthesize_pinned_buf_gen_yaml,
)
from pants.backend.codegen.protobuf.buf.subsystem import BufSubsystem
from pants.backend.codegen.protobuf.protoc import Protoc
from pants.backend.codegen.protobuf.python.additional_fields import PythonSourceRootField
from pants.backend.codegen.protobuf.target_types import ProtobufSourceField
from pants.core.util_rules.config_files import find_config_file
from pants.core.util_rules.external_tool import download_external_tool
from pants.core.util_rules.source_files import SourceFilesRequest, determine_source_files
from pants.engine.fs import CreateDigest, Directory, FileContent, MergeDigests, RemovePrefix
from pants.engine.internals.graph import transitive_targets as transitive_targets_get
from pants.engine.intrinsics import (
    create_digest,
    digest_to_snapshot,
    get_digest_contents,
    merge_digests,
    remove_prefix,
)
from pants.engine.platform import Platform
from pants.engine.process import Process, execute_process_or_raise
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
from pants.engine.target import GeneratedSources, Target, TransitiveTargetsRequest
from pants.util.logging import LogLevel

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class GeneratePythonFromProtobufViaBufRequest:
    protocol_target: Target


@rule(desc="Generate Python from Protobuf via `buf generate`", level=LogLevel.DEBUG)
async def generate_python_from_protobuf_via_buf(
    request: GeneratePythonFromProtobufViaBufRequest,
    buf: BufSubsystem,
    protoc: Protoc,
    platform: Platform,
) -> GeneratedSources:
    target = request.protocol_target

    if target.get(PythonSourceRootField).value is not None:
        logger.warning(
            "`python_source_root` is set on %s but `protobuf_generator='buf'`; "
            "the field is ignored — output paths come from the `out:` field of "
            "`buf.gen.yaml`.",
            target.address,
        )

    output_dir = "_generated_files"
    create_output_dir_request = create_digest(CreateDigest([Directory(output_dir)]))

    # Buf needs all transitive `.proto` sources to resolve imports, even though only
    # the target's own files are passed via `--path`.
    transitive_targets = await transitive_targets_get(
        TransitiveTargetsRequest([target.address]), **implicitly()
    )

    # Unlike the protoc path, buf operates on original (unstripped) paths because
    # the buf module root is determined by `buf.yaml`'s location, not by Pants source
    # roots.
    all_sources_request = determine_source_files(
        SourceFilesRequest(
            tgt[ProtobufSourceField]
            for tgt in transitive_targets.closure
            if tgt.has_field(ProtobufSourceField)
        )
    )
    target_sources_request = determine_source_files(
        SourceFilesRequest([target[ProtobufSourceField]])
    )

    download_buf_request = download_external_tool(buf.get_request(platform))
    download_protoc_request = download_external_tool(protoc.get_request(platform))
    config_files_request = find_config_file(buf.config_request)
    gen_template_files_request = find_config_file(gen_template_request_for_target(target, buf))

    (
        downloaded_buf,
        downloaded_protoc,
        empty_output_dir,
        all_sources,
        target_sources,
        config_files,
        gen_template_files,
    ) = await concurrently(
        download_buf_request,
        download_protoc_request,
        create_output_dir_request,
        all_sources_request,
        target_sources_request,
        config_files_request,
        gen_template_files_request,
    )

    # If the user's `buf.yaml` declares BSR `deps:`, require a sibling
    # `buf.lock` so codegen is reproducible. The lock is what pins each dep to
    # an exact commit; without it, buf would resolve to whatever is currently
    # latest on the BSR.
    config_yaml_paths = [
        p for p in config_files.snapshot.files if os.path.basename(p) == "buf.yaml"
    ]
    config_lock_paths = {
        p for p in config_files.snapshot.files if os.path.basename(p) == "buf.lock"
    }
    if config_yaml_paths:
        yaml_path = config_yaml_paths[0]
        config_dcs = await get_digest_contents(config_files.snapshot.digest)
        yaml_content = next(
            (dc.content for dc in config_dcs if dc.path == yaml_path),
            b"",
        )
        deps = parse_buf_yaml_deps(yaml_content)
        expected_lock = os.path.join(os.path.dirname(yaml_path), "buf.lock")
        if deps and expected_lock not in config_lock_paths:
            resolve_name = os.path.dirname(yaml_path) or "buf"
            raise MissingBufLockError(
                f"`{yaml_path}` declares `deps:` ({', '.join(deps)}) but no "
                f"`{expected_lock}` was found. Pants requires a `buf.lock` so "
                f"BSR deps are pinned and codegen is reproducible.\n\n"
                f"Run `pants generate-lockfiles --resolve={resolve_name}` to "
                f"create it."
            )

    # Resolve every `remote:` plugin to an exact `:vX.Y:revN` pin before
    # invoking buf. Unpinned entries that can be filled in from
    # `DEFAULT_PLUGIN_PINS` (or the user's `[buf].extra_plugin_pins`) get a
    # default; unknown unpinned entries raise. The resulting yaml is written
    # into a fresh digest that replaces the user's `buf.gen.yaml` in the
    # sandbox, so buf sees a hermetic, fully-pinned config.
    gen_template_digest = gen_template_files.snapshot.digest
    if gen_template_files.snapshot.files:
        gen_template_path = gen_template_files.snapshot.files[0]
        gen_template_dcs = await get_digest_contents(gen_template_digest)
        gen_template_content = next(
            (dc.content for dc in gen_template_dcs if dc.path == gen_template_path),
            b"",
        )
        synthesized = synthesize_pinned_buf_gen_yaml(
            gen_template_content,
            gen_template_path,
            extra_pins=buf.extra_plugin_pins,
        )
        if synthesized != gen_template_content:
            gen_template_digest = await create_digest(
                CreateDigest(
                    [FileContent(gen_template_path, synthesized)],
                )
            )

    input_digest = await merge_digests(
        MergeDigests(
            (
                all_sources.snapshot.digest,
                empty_output_dir,
                downloaded_buf.digest,
                config_files.snapshot.digest,
                gen_template_digest,
            )
        )
    )

    config_arg = ["--config", buf.config] if buf.config else []
    template_path = resolved_template_path(target, buf)
    template_arg = ["--template", template_path] if template_path else []

    argv = [
        downloaded_buf.exe,
        "generate",
        *config_arg,
        *template_arg,
        "--output",
        output_dir,
        *buf.gen_args,
        "--path",
        ",".join(target_sources.snapshot.files),
    ]

    # Expose `protoc` (and any plugin binaries co-located with it) on PATH so
    # `buf generate` can resolve `protoc_builtin:` and `local: [protoc]` plugin
    # entries.
    protoc_relpath = "__protoc"
    protoc_bin_dir = os.path.join(protoc_relpath, os.path.dirname(downloaded_protoc.exe))

    result = await execute_process_or_raise(
        **implicitly(
            Process(
                argv=argv,
                input_digest=input_digest,
                immutable_input_digests={protoc_relpath: downloaded_protoc.digest},
                env={"PATH": protoc_bin_dir},
                description=f"Generating Python from Protobuf via buf for {target.address}.",
                level=LogLevel.DEBUG,
                output_directories=(output_dir,),
            )
        ),
    )

    # Strip the sandbox `output_dir` prefix; the buf.gen.yaml's `out:` paths land at
    # exactly the locations the user declared.
    normalized = await remove_prefix(RemovePrefix(result.output_digest, output_dir))
    snapshot = await digest_to_snapshot(normalized)
    return GeneratedSources(snapshot)


def rules():
    return collect_rules()

1	# Copyright 2026 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	6✔
5
6	import logging	6✔
7	import os	6✔
8	from dataclasses import dataclass	6✔
9
10	from pants.backend.codegen.protobuf.buf.config import (	6✔
11	MissingBufLockError,
12	gen_template_request_for_target,
13	parse_buf_yaml_deps,
14	resolved_template_path,
15	synthesize_pinned_buf_gen_yaml,
16	)
17	from pants.backend.codegen.protobuf.buf.subsystem import BufSubsystem	6✔
18	from pants.backend.codegen.protobuf.protoc import Protoc	6✔
19	from pants.backend.codegen.protobuf.python.additional_fields import PythonSourceRootField	6✔
20	from pants.backend.codegen.protobuf.target_types import ProtobufSourceField	6✔
21	from pants.core.util_rules.config_files import find_config_file	6✔
22	from pants.core.util_rules.external_tool import download_external_tool	6✔
23	from pants.core.util_rules.source_files import SourceFilesRequest, determine_source_files	6✔
24	from pants.engine.fs import CreateDigest, Directory, FileContent, MergeDigests, RemovePrefix	6✔
25	from pants.engine.internals.graph import transitive_targets as transitive_targets_get	6✔
26	from pants.engine.intrinsics import (	6✔
27	create_digest,
28	digest_to_snapshot,
29	get_digest_contents,
30	merge_digests,
31	remove_prefix,
32	)
33	from pants.engine.platform import Platform	6✔
34	from pants.engine.process import Process, execute_process_or_raise	6✔
35	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	6✔
36	from pants.engine.target import GeneratedSources, Target, TransitiveTargetsRequest	6✔
37	from pants.util.logging import LogLevel	6✔
38
39	logger = logging.getLogger(__name__)	6✔
40
41
42	@dataclass(frozen=True)	6✔
43	class GeneratePythonFromProtobufViaBufRequest:	6✔
44	protocol_target: Target	6✔
45
46
47	@rule(desc="Generate Python from Protobuf via `buf generate`", level=LogLevel.DEBUG)	6✔
48	async def generate_python_from_protobuf_via_buf(	6✔
49	request: GeneratePythonFromProtobufViaBufRequest,
50	buf: BufSubsystem,
51	protoc: Protoc,
52	platform: Platform,
53	) -> GeneratedSources:
54	target = request.protocol_target	3✔
55
56	if target.get(PythonSourceRootField).value is not None:	3✔
NEW 57	logger.warning(	×
58	"`python_source_root` is set on %s but `protobuf_generator='buf'`; "
59	"the field is ignored — output paths come from the `out:` field of "
60	"`buf.gen.yaml`.",
61	target.address,
62	)
63
64	output_dir = "_generated_files"	3✔
65	create_output_dir_request = create_digest(CreateDigest([Directory(output_dir)]))	3✔
66
67	# Buf needs all transitive `.proto` sources to resolve imports, even though only
68	# the target's own files are passed via `--path`.
69	transitive_targets = await transitive_targets_get(	3✔
70	TransitiveTargetsRequest([target.address]), **implicitly()
71	)
72
73	# Unlike the protoc path, buf operates on original (unstripped) paths because
74	# the buf module root is determined by `buf.yaml`'s location, not by Pants source
75	# roots.
76	all_sources_request = determine_source_files(	3✔
77	SourceFilesRequest(
78	tgt[ProtobufSourceField]
79	for tgt in transitive_targets.closure
80	if tgt.has_field(ProtobufSourceField)
81	)
82	)
83	target_sources_request = determine_source_files(	3✔
84	SourceFilesRequest([target[ProtobufSourceField]])
85	)
86
87	download_buf_request = download_external_tool(buf.get_request(platform))	3✔
88	download_protoc_request = download_external_tool(protoc.get_request(platform))	3✔
89	config_files_request = find_config_file(buf.config_request)	3✔
90	gen_template_files_request = find_config_file(gen_template_request_for_target(target, buf))	3✔
91
92	(	3✔
93	downloaded_buf,
94	downloaded_protoc,
95	empty_output_dir,
96	all_sources,
97	target_sources,
98	config_files,
99	gen_template_files,
100	) = await concurrently(
101	download_buf_request,
102	download_protoc_request,
103	create_output_dir_request,
104	all_sources_request,
105	target_sources_request,
106	config_files_request,
107	gen_template_files_request,
108	)
109
110	# If the user's `buf.yaml` declares BSR `deps:`, require a sibling
111	# `buf.lock` so codegen is reproducible. The lock is what pins each dep to
112	# an exact commit; without it, buf would resolve to whatever is currently
113	# latest on the BSR.
114	config_yaml_paths = [	3✔
115	p for p in config_files.snapshot.files if os.path.basename(p) == "buf.yaml"
116	]
117	config_lock_paths = {	3✔
118	p for p in config_files.snapshot.files if os.path.basename(p) == "buf.lock"
119	}
120	if config_yaml_paths:	3✔
121	yaml_path = config_yaml_paths[0]	3✔
122	config_dcs = await get_digest_contents(config_files.snapshot.digest)	3✔
123	yaml_content = next(	3✔
124	(dc.content for dc in config_dcs if dc.path == yaml_path),
125	b"",
126	)
127	deps = parse_buf_yaml_deps(yaml_content)	3✔
128	expected_lock = os.path.join(os.path.dirname(yaml_path), "buf.lock")	3✔
129	if deps and expected_lock not in config_lock_paths:	3✔
130	resolve_name = os.path.dirname(yaml_path) or "buf"	3✔
131	raise MissingBufLockError(	3✔
132	f"`{yaml_path}` declares `deps:` ({', '.join(deps)}) but no "
133	f"`{expected_lock}` was found. Pants requires a `buf.lock` so "
134	f"BSR deps are pinned and codegen is reproducible.\n\n"
135	f"Run `pants generate-lockfiles --resolve={resolve_name}` to "
136	f"create it."
137	)
138
139	# Resolve every `remote:` plugin to an exact `:vX.Y:revN` pin before
140	# invoking buf. Unpinned entries that can be filled in from
141	# `DEFAULT_PLUGIN_PINS` (or the user's `[buf].extra_plugin_pins`) get a
142	# default; unknown unpinned entries raise. The resulting yaml is written
143	# into a fresh digest that replaces the user's `buf.gen.yaml` in the
144	# sandbox, so buf sees a hermetic, fully-pinned config.
145	gen_template_digest = gen_template_files.snapshot.digest	3✔
146	if gen_template_files.snapshot.files:	3✔
147	gen_template_path = gen_template_files.snapshot.files[0]	3✔
148	gen_template_dcs = await get_digest_contents(gen_template_digest)	3✔
149	gen_template_content = next(	3✔
150	(dc.content for dc in gen_template_dcs if dc.path == gen_template_path),
151	b"",
152	)
153	synthesized = synthesize_pinned_buf_gen_yaml(	3✔
154	gen_template_content,
155	gen_template_path,
156	extra_pins=buf.extra_plugin_pins,
157	)
158	if synthesized != gen_template_content:	3✔
159	gen_template_digest = await create_digest(	3✔
160	CreateDigest(
161	[FileContent(gen_template_path, synthesized)],
162	)
163	)
164
165	input_digest = await merge_digests(	3✔
166	MergeDigests(
167	(
168	all_sources.snapshot.digest,
169	empty_output_dir,
170	downloaded_buf.digest,
171	config_files.snapshot.digest,
172	gen_template_digest,
173	)
174	)
175	)
176
177	config_arg = ["--config", buf.config] if buf.config else []	3✔
178	template_path = resolved_template_path(target, buf)	3✔
179	template_arg = ["--template", template_path] if template_path else []	3✔
180
181	argv = [	3✔
182	downloaded_buf.exe,
183	"generate",
184	*config_arg,
185	*template_arg,
186	"--output",
187	output_dir,
188	*buf.gen_args,
189	"--path",
190	",".join(target_sources.snapshot.files),
191	]
192
193	# Expose `protoc` (and any plugin binaries co-located with it) on PATH so
194	# `buf generate` can resolve `protoc_builtin:` and `local: [protoc]` plugin
195	# entries.
196	protoc_relpath = "__protoc"	3✔
197	protoc_bin_dir = os.path.join(protoc_relpath, os.path.dirname(downloaded_protoc.exe))	3✔
198
199	result = await execute_process_or_raise(	3✔
200	**implicitly(
201	Process(
202	argv=argv,
203	input_digest=input_digest,
204	immutable_input_digests={protoc_relpath: downloaded_protoc.digest},
205	env={"PATH": protoc_bin_dir},
206	description=f"Generating Python from Protobuf via buf for {target.address}.",
207	level=LogLevel.DEBUG,
208	output_directories=(output_dir,),
209	)
210	),
211	)
212
213	# Strip the sandbox `output_dir` prefix; the buf.gen.yaml's `out:` paths land at
214	# exactly the locations the user declared.
215	normalized = await remove_prefix(RemovePrefix(result.output_digest, output_dir))	3✔
216	snapshot = await digest_to_snapshot(normalized)	3✔
217	return GeneratedSources(snapshot)	3✔
218
219
220	def rules():	6✔
221	return collect_rules()	6✔

pantsbuild / pants / 26689585807

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous