26689585807

Committed 30 May 2026 04:55PM UTC coverage: 92.742% (-0.05%) from 92.792%

Build # 26689585807

Build Type

Pull #23343

github

Committed by

web-flow

Commit Message

Merge c42efe377 into c8127c1f4

Pull Request Pull Request #23343: Add buf as an alternate Python protobuf code generator

Coverage Stats

767 of 807 new or added lines in 17 files covered. (95.04%)

69 existing lines in 3 files now uncovered.

93753 of 101090 relevant lines covered (92.74%)

4.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.0

/src/python/pants/backend/codegen/protobuf/buf/lockfile.py

# Copyright 2026 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

"""Generate `buf.lock` for buf-managed proto modules.

Each `buf.yaml` in the repo defines a separate buf module (resolve). Running
`pants generate-lockfiles` invokes `buf dep update` for each, producing a
fully-pinned `buf.lock` next to the corresponding `buf.yaml`.
"""

from __future__ import annotations

import os
from dataclasses import dataclass

from pants.backend.codegen.protobuf.buf.subsystem import BufSubsystem
from pants.core.goals.generate_lockfiles import (
    GenerateLockfile,
    GenerateLockfileResult,
    KnownUserResolveNames,
    KnownUserResolveNamesRequest,
    RequestedUserResolveNames,
    UserGenerateLockfiles,
)
from pants.core.util_rules.config_files import find_config_file
from pants.core.util_rules.external_tool import download_external_tool
from pants.engine.fs import MergeDigests, PathGlobs
from pants.engine.intrinsics import merge_digests, path_globs_to_digest
from pants.engine.platform import Platform
from pants.engine.process import Process, execute_process_or_raise
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
from pants.engine.unions import UnionRule
from pants.util.logging import LogLevel


class KnownBufResolveNamesRequest(KnownUserResolveNamesRequest):
    pass


class RequestedBufResolveNames(RequestedUserResolveNames):
    pass


@dataclass(frozen=True)
class GenerateBufLockfile(GenerateLockfile):
    """A request to (re)generate `buf.lock` for a single buf module."""

    buf_yaml_path: str


def _resolve_name(buf_yaml_path: str) -> str:
    parent = os.path.dirname(buf_yaml_path)
    return parent if parent else "buf"


@rule
async def known_buf_user_resolve_names(
    _: KnownBufResolveNamesRequest, buf: BufSubsystem
) -> KnownUserResolveNames:
    files = await find_config_file(buf.config_request)
    yaml_paths = sorted(p for p in files.snapshot.files if os.path.basename(p) == "buf.yaml")
    return KnownUserResolveNames(
        names=tuple(_resolve_name(p) for p in yaml_paths),
        option_name="`buf.yaml` discovery",
        requested_resolve_names_cls=RequestedBufResolveNames,
    )


@rule
async def setup_user_buf_lockfile_requests(
    requested: RequestedBufResolveNames, buf: BufSubsystem
) -> UserGenerateLockfiles:
    files = await find_config_file(buf.config_request)
    name_to_yaml = {
        _resolve_name(p): p for p in files.snapshot.files if os.path.basename(p) == "buf.yaml"
    }
    requests = []
    for name in requested:
        path = name_to_yaml.get(name)
        if path is None:
            continue
        requests.append(
            GenerateBufLockfile(
                resolve_name=name,
                lockfile_dest=os.path.join(os.path.dirname(path), "buf.lock"),
                diff=False,
                buf_yaml_path=path,
            )
        )
    return UserGenerateLockfiles(requests)


@rule(desc="Resolve buf.yaml deps via `buf dep update`", level=LogLevel.DEBUG)
async def generate_buf_lockfile(
    req: GenerateBufLockfile, buf: BufSubsystem, platform: Platform
) -> GenerateLockfileResult:
    buf_yaml_dir = os.path.dirname(req.buf_yaml_path) or "."

    # `buf dep update` does a partial build, so it needs at least one `.proto`
    # file in the module to operate. Glob them in alongside the buf.yaml/buf.lock.
    proto_glob = f"{buf_yaml_dir}/**/*.proto" if buf_yaml_dir != "." else "**/*.proto"
    downloaded_buf, files, protos_digest = await concurrently(
        download_external_tool(buf.get_request(platform)),
        find_config_file(buf.config_request),
        path_globs_to_digest(PathGlobs([proto_glob])),
    )

    input_digest = await merge_digests(
        MergeDigests((files.snapshot.digest, protos_digest, downloaded_buf.digest))
    )

    process_result = await execute_process_or_raise(
        **implicitly(
            Process(
                argv=[downloaded_buf.exe, "dep", "update", buf_yaml_dir],
                input_digest=input_digest,
                description=f"Resolving buf.lock for `{req.resolve_name}`",
                level=LogLevel.DEBUG,
                output_files=(req.lockfile_dest,),
            )
        )
    )

    return GenerateLockfileResult(
        digest=process_result.output_digest,
        resolve_name=req.resolve_name,
        path=req.lockfile_dest,
    )


def rules():
    return [
        *collect_rules(),
        UnionRule(GenerateLockfile, GenerateBufLockfile),
        UnionRule(KnownUserResolveNamesRequest, KnownBufResolveNamesRequest),
        UnionRule(RequestedUserResolveNames, RequestedBufResolveNames),
    ]

1	# Copyright 2026 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	"""Generate `buf.lock` for buf-managed proto modules.
5
6	Each `buf.yaml` in the repo defines a separate buf module (resolve). Running
7	`pants generate-lockfiles` invokes `buf dep update` for each, producing a
8	fully-pinned `buf.lock` next to the corresponding `buf.yaml`.
9	"""
10
11	from __future__ import annotations	1✔
12
13	import os	1✔
14	from dataclasses import dataclass	1✔
15
16	from pants.backend.codegen.protobuf.buf.subsystem import BufSubsystem	1✔
17	from pants.core.goals.generate_lockfiles import (	1✔
18	GenerateLockfile,
19	GenerateLockfileResult,
20	KnownUserResolveNames,
21	KnownUserResolveNamesRequest,
22	RequestedUserResolveNames,
23	UserGenerateLockfiles,
24	)
25	from pants.core.util_rules.config_files import find_config_file	1✔
26	from pants.core.util_rules.external_tool import download_external_tool	1✔
27	from pants.engine.fs import MergeDigests, PathGlobs	1✔
28	from pants.engine.intrinsics import merge_digests, path_globs_to_digest	1✔
29	from pants.engine.platform import Platform	1✔
30	from pants.engine.process import Process, execute_process_or_raise	1✔
31	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	1✔
32	from pants.engine.unions import UnionRule	1✔
33	from pants.util.logging import LogLevel	1✔
34
35
36	class KnownBufResolveNamesRequest(KnownUserResolveNamesRequest):	1✔
37	pass	1✔
38
39
40	class RequestedBufResolveNames(RequestedUserResolveNames):	1✔
41	pass	1✔
42
43
44	@dataclass(frozen=True)	1✔
45	class GenerateBufLockfile(GenerateLockfile):	1✔
46	"""A request to (re)generate `buf.lock` for a single buf module."""
47
48	buf_yaml_path: str	1✔
49
50
51	def _resolve_name(buf_yaml_path: str) -> str:	1✔
52	parent = os.path.dirname(buf_yaml_path)	1✔
53	return parent if parent else "buf"	1✔
54
55
56	@rule	1✔
57	async def known_buf_user_resolve_names(	1✔
58	_: KnownBufResolveNamesRequest, buf: BufSubsystem
59	) -> KnownUserResolveNames:
60	files = await find_config_file(buf.config_request)	1✔
61	yaml_paths = sorted(p for p in files.snapshot.files if os.path.basename(p) == "buf.yaml")	1✔
62	return KnownUserResolveNames(	1✔
63	names=tuple(_resolve_name(p) for p in yaml_paths),
64	option_name="`buf.yaml` discovery",
65	requested_resolve_names_cls=RequestedBufResolveNames,
66	)
67
68
69	@rule	1✔
70	async def setup_user_buf_lockfile_requests(	1✔
71	requested: RequestedBufResolveNames, buf: BufSubsystem
72	) -> UserGenerateLockfiles:
73	files = await find_config_file(buf.config_request)	1✔
74	name_to_yaml = {	1✔
75	_resolve_name(p): p for p in files.snapshot.files if os.path.basename(p) == "buf.yaml"
76	}
77	requests = []	1✔
78	for name in requested:	1✔
79	path = name_to_yaml.get(name)	1✔
80	if path is None:	1✔
81	continue	1✔
82	requests.append(	1✔
83	GenerateBufLockfile(
84	resolve_name=name,
85	lockfile_dest=os.path.join(os.path.dirname(path), "buf.lock"),
86	diff=False,
87	buf_yaml_path=path,
88	)
89	)
90	return UserGenerateLockfiles(requests)	1✔
91
92
93	@rule(desc="Resolve buf.yaml deps via `buf dep update`", level=LogLevel.DEBUG)	1✔
94	async def generate_buf_lockfile(	1✔
95	req: GenerateBufLockfile, buf: BufSubsystem, platform: Platform
96	) -> GenerateLockfileResult:
NEW 97	buf_yaml_dir = os.path.dirname(req.buf_yaml_path) or "."	×
98
99	# `buf dep update` does a partial build, so it needs at least one `.proto`
100	# file in the module to operate. Glob them in alongside the buf.yaml/buf.lock.
NEW 101	proto_glob = f"{buf_yaml_dir}/*/.proto" if buf_yaml_dir != "." else "*/.proto"	×
NEW 102	downloaded_buf, files, protos_digest = await concurrently(	×
103	download_external_tool(buf.get_request(platform)),
104	find_config_file(buf.config_request),
105	path_globs_to_digest(PathGlobs([proto_glob])),
106	)
107
NEW 108	input_digest = await merge_digests(	×
109	MergeDigests((files.snapshot.digest, protos_digest, downloaded_buf.digest))
110	)
111
NEW 112	process_result = await execute_process_or_raise(	×
113	**implicitly(
114	Process(
115	argv=[downloaded_buf.exe, "dep", "update", buf_yaml_dir],
116	input_digest=input_digest,
117	description=f"Resolving buf.lock for `{req.resolve_name}`",
118	level=LogLevel.DEBUG,
119	output_files=(req.lockfile_dest,),
120	)
121	)
122	)
123
NEW 124	return GenerateLockfileResult(	×
125	digest=process_result.output_digest,
126	resolve_name=req.resolve_name,
127	path=req.lockfile_dest,
128	)
129
130
131	def rules():	1✔
132	return [	1✔
133	*collect_rules(),
134	UnionRule(GenerateLockfile, GenerateBufLockfile),
135	UnionRule(KnownUserResolveNamesRequest, KnownBufResolveNamesRequest),
136	UnionRule(RequestedUserResolveNames, RequestedBufResolveNames),
137	]

pantsbuild / pants / 26689585807

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous