20332790708

Committed 18 Dec 2025 09:48AM UTC coverage: 64.992% (-15.3%) from 80.295%

Build # 20332790708

Build Type

Pull #22949

github

Committed by

web-flow

Commit Message

Merge f730a56cd into 407284c67

Pull Request Pull Request #22949: Add experimental uv resolver for Python lockfiles

Run Details

54 of 97 new or added lines in 5 files covered. (55.67%)

8270 existing lines in 295 files now uncovered.

48990 of 75379 relevant lines covered (64.99%)

1.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/python/pants/backend/codegen/protobuf/scala/rules.py

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
UNCOV 3	from __future__ import annotations	×
4
UNCOV 5	import os	×
UNCOV 6	from dataclasses import dataclass	×
7
UNCOV 8	from pants.backend.codegen.protobuf import protoc	×
UNCOV 9	from pants.backend.codegen.protobuf.protoc import Protoc	×
UNCOV 10	from pants.backend.codegen.protobuf.scala import dependency_inference, symbol_mapper	×
UNCOV 11	from pants.backend.codegen.protobuf.scala.subsystem import PluginArtifactSpec, ScalaPBSubsystem	×
UNCOV 12	from pants.backend.codegen.protobuf.target_types import (	×
13	ProtobufSourceField,
14	ProtobufSourcesGeneratorTarget,
15	ProtobufSourceTarget,
16	)
UNCOV 17	from pants.backend.scala.target_types import ScalaSourceField	×
UNCOV 18	from pants.backend.scala.util_rules.versions import (	×
19	ScalaArtifactsForVersionRequest,
20	ScalaVersion,
21	resolve_scala_artifacts_for_version,
22	)
UNCOV 23	from pants.core.goals.resolves import ExportableTool	×
UNCOV 24	from pants.core.util_rules import distdir	×
UNCOV 25	from pants.core.util_rules.env_vars import environment_vars_subset	×
UNCOV 26	from pants.core.util_rules.external_tool import download_external_tool	×
UNCOV 27	from pants.core.util_rules.source_files import SourceFilesRequest	×
UNCOV 28	from pants.core.util_rules.stripped_source_files import strip_source_roots	×
UNCOV 29	from pants.engine.env_vars import EnvironmentVarsRequest	×
UNCOV 30	from pants.engine.fs import (	×
31	AddPrefix,
32	CreateDigest,
33	Digest,
34	Directory,
35	FileContent,
36	MergeDigests,
37	RemovePrefix,
38	)
UNCOV 39	from pants.engine.internals.graph import transitive_targets	×
UNCOV 40	from pants.engine.internals.native_engine import EMPTY_DIGEST	×
UNCOV 41	from pants.engine.internals.selectors import concurrently	×
UNCOV 42	from pants.engine.intrinsics import (	×
43	add_prefix,
44	create_digest,
45	digest_to_snapshot,
46	merge_digests,
47	remove_prefix,
48	)
UNCOV 49	from pants.engine.platform import Platform	×
UNCOV 50	from pants.engine.process import fallible_to_exec_result_or_raise	×
UNCOV 51	from pants.engine.rules import collect_rules, implicitly, rule	×
UNCOV 52	from pants.engine.target import GeneratedSources, GenerateSourcesRequest, TransitiveTargetsRequest	×
UNCOV 53	from pants.engine.unions import UnionRule	×
UNCOV 54	from pants.jvm.compile import ClasspathEntry	×
UNCOV 55	from pants.jvm.dependency_inference import artifact_mapper	×
UNCOV 56	from pants.jvm.goals import lockfile	×
UNCOV 57	from pants.jvm.jdk_rules import InternalJdk, JvmProcess	×
UNCOV 58	from pants.jvm.resolve.common import ArtifactRequirements, GatherJvmCoordinatesRequest	×
UNCOV 59	from pants.jvm.resolve.coursier_fetch import (	×
60	ToolClasspath,
61	ToolClasspathRequest,
62	materialize_classpath_for_tool,
63	)
UNCOV 64	from pants.jvm.resolve.jvm_tool import (	×
65	GenerateJvmLockfileFromTool,
66	gather_coordinates_for_jvm_lockfile,
67	)
UNCOV 68	from pants.jvm.target_types import PrefixedJvmJdkField, PrefixedJvmResolveField	×
UNCOV 69	from pants.source.source_root import SourceRootRequest, get_source_root	×
UNCOV 70	from pants.util.logging import LogLevel	×
UNCOV 71	from pants.util.ordered_set import FrozenOrderedSet	×
UNCOV 72	from pants.util.resources import read_resource	×
73
74
UNCOV 75	class GenerateScalaFromProtobufRequest(GenerateSourcesRequest):	×
UNCOV 76	input = ProtobufSourceField	×
UNCOV 77	output = ScalaSourceField	×
78
79
UNCOV 80	class ScalaPBShimCompiledClassfiles(ClasspathEntry):	×
UNCOV 81	pass	×
82
83
UNCOV 84	@dataclass(frozen=True)	×
UNCOV 85	class MaterializeJvmPluginRequest:	×
UNCOV 86	plugin: PluginArtifactSpec	×
87
88
UNCOV 89	@dataclass(frozen=True)	×
UNCOV 90	class MaterializedJvmPlugin:	×
UNCOV 91	name: str	×
UNCOV 92	classpath: ToolClasspath	×
93
UNCOV 94	def setup_arg(self, plugin_relpath: str) -> str:	×
95	classpath_arg = ":".join(self.classpath.classpath_entries(plugin_relpath))	×
96	return f"--jvm-plugin={self.name}={classpath_arg}"	×
97
98
UNCOV 99	@dataclass(frozen=True)	×
UNCOV 100	class MaterializeJvmPluginsRequest:	×
UNCOV 101	plugins: tuple[PluginArtifactSpec, ...]	×
102
103
UNCOV 104	@dataclass(frozen=True)	×
UNCOV 105	class MaterializedJvmPlugins:	×
UNCOV 106	digest: Digest	×
UNCOV 107	plugins: tuple[MaterializedJvmPlugin, ...]	×
108
UNCOV 109	def setup_args(self, plugins_relpath: str) -> tuple[str, ...]:	×
110	return tuple(p.setup_arg(os.path.join(plugins_relpath, p.name)) for p in self.plugins)	×
111
112
UNCOV 113	@rule	×
UNCOV 114	async def materialize_jvm_plugin(request: MaterializeJvmPluginRequest) -> MaterializedJvmPlugin:	×
115	requirements = await gather_coordinates_for_jvm_lockfile(	×
116	GatherJvmCoordinatesRequest(
117	artifact_inputs=FrozenOrderedSet([request.plugin.artifact]),
118	option_name="[scalapb].jvm_plugins",
119	)
120	)
121	classpath = await materialize_classpath_for_tool(	×
122	ToolClasspathRequest(artifact_requirements=requirements)
123	)
124	return MaterializedJvmPlugin(name=request.plugin.name, classpath=classpath)	×
125
126
UNCOV 127	@rule	×
UNCOV 128	async def materialize_jvm_plugins(	×
129	request: MaterializeJvmPluginsRequest,
130	) -> MaterializedJvmPlugins:
131	materialized_plugins = await concurrently(	×
132	materialize_jvm_plugin(MaterializeJvmPluginRequest(plugin)) for plugin in request.plugins
133	)
134	plugin_digests = await concurrently(	×
135	add_prefix(AddPrefix(p.classpath.digest, p.name)) for p in materialized_plugins
136	)
137	merged_plugins_digest = await merge_digests(MergeDigests(plugin_digests))	×
138	return MaterializedJvmPlugins(merged_plugins_digest, materialized_plugins)	×
139
140
UNCOV 141	@rule(desc="Generate Scala from Protobuf", level=LogLevel.DEBUG)	×
UNCOV 142	async def generate_scala_from_protobuf(	×
143	request: GenerateScalaFromProtobufRequest,
144	protoc: Protoc,
145	scalapb: ScalaPBSubsystem,
146	shim_classfiles: ScalaPBShimCompiledClassfiles,
147	jdk: InternalJdk,
148	platform: Platform,
149	) -> GeneratedSources:
150	output_dir = "_generated_files"	×
151	toolcp_relpath = "__toolcp"	×
152	shimcp_relpath = "__shimcp"	×
153	plugins_relpath = "__plugins"	×
154	protoc_relpath = "__protoc"	×
155
156	lockfile_request = GenerateJvmLockfileFromTool.create(scalapb)	×
157	(	×
158	downloaded_protoc_binary,
159	tool_classpath,
160	empty_output_dir,
161	transitive_targets_for_protobuf_source,
162	inherit_env,
163	) = await concurrently(
164	download_external_tool(protoc.get_request(platform)),
165	materialize_classpath_for_tool(ToolClasspathRequest(lockfile=lockfile_request)),
166	create_digest(CreateDigest([Directory(output_dir)])),
167	transitive_targets(
168	TransitiveTargetsRequest([request.protocol_target.address]), **implicitly()
169	),
170	# Need PATH so that ScalaPB can invoke `mkfifo`.
171	environment_vars_subset(EnvironmentVarsRequest(requested=["PATH"]), **implicitly()),
172	)
173
174	# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
175	# for Protobuf imports to be discoverable.
176	all_sources_stripped, target_sources_stripped = await concurrently(	×
177	strip_source_roots(
178	**implicitly(
179	SourceFilesRequest(
180	tgt[ProtobufSourceField]
181	for tgt in transitive_targets_for_protobuf_source.closure
182	if tgt.has_field(ProtobufSourceField)
183	)
184	)
185	),
186	strip_source_roots(
187	**implicitly(SourceFilesRequest([request.protocol_target[ProtobufSourceField]]))
188	),
189	)
190
191	merged_jvm_plugins_digest = EMPTY_DIGEST	×
192	maybe_jvm_plugins_setup_args: tuple[str, ...] = ()	×
193	maybe_jvm_plugins_output_args: tuple[str, ...] = ()	×
194	jvm_plugins = scalapb.jvm_plugins	×
195	if jvm_plugins:	×
196	materialized_jvm_plugins = await materialize_jvm_plugins(	×
197	MaterializeJvmPluginsRequest(jvm_plugins)
198	)
199	merged_jvm_plugins_digest = materialized_jvm_plugins.digest	×
200	maybe_jvm_plugins_setup_args = materialized_jvm_plugins.setup_args(plugins_relpath)	×
201	maybe_jvm_plugins_output_args = tuple(	×
202	f"--{plugin.name}_out={output_dir}" for plugin in materialized_jvm_plugins.plugins
203	)
204
205	extra_immutable_input_digests = {	×
206	toolcp_relpath: tool_classpath.digest,
207	shimcp_relpath: shim_classfiles.digest,
208	plugins_relpath: merged_jvm_plugins_digest,
209	protoc_relpath: downloaded_protoc_binary.digest,
210	}
211
212	input_digest = await merge_digests(	×
213	MergeDigests([all_sources_stripped.snapshot.digest, empty_output_dir])
214	)
215
216	result = await fallible_to_exec_result_or_raise(	×
217	**implicitly(
218	JvmProcess(
219	jdk=jdk,
220	classpath_entries=[
221	*tool_classpath.classpath_entries(toolcp_relpath),
222	shimcp_relpath,
223	],
224	argv=[
225	"org.pantsbuild.backend.scala.scalapb.ScalaPBShim",
226	f"--protoc={os.path.join(protoc_relpath, downloaded_protoc_binary.exe)}",
227	*maybe_jvm_plugins_setup_args,
228	f"--scala_out={output_dir}",
229	*maybe_jvm_plugins_output_args,
230	*target_sources_stripped.snapshot.files,
231	],
232	input_digest=input_digest,
233	extra_immutable_input_digests=extra_immutable_input_digests,
234	extra_nailgun_keys=extra_immutable_input_digests,
235	description=f"Generating Scala sources from {request.protocol_target.address}.",
236	level=LogLevel.DEBUG,
237	output_directories=(output_dir,),
238	extra_env=inherit_env,
239	)
240	)
241	)
242
243	normalized_digest, source_root = await concurrently(	×
244	remove_prefix(RemovePrefix(result.output_digest, output_dir)),
245	get_source_root(SourceRootRequest.for_target(request.protocol_target)),
246	)
247
248	source_root_restored = (	×
249	await digest_to_snapshot(**implicitly(AddPrefix(normalized_digest, source_root.path)))
250	if source_root.path != "."
251	else await digest_to_snapshot(normalized_digest)
252	)
253	return GeneratedSources(source_root_restored)	×
254
255
UNCOV 256	SHIM_SCALA_VERSION = ScalaVersion.parse("2.13.7")	×
257
258
259	# TODO(13879): Consolidate compilation of wrapper binaries to common rules.
UNCOV 260	@rule	×
UNCOV 261	async def setup_scalapb_shim_classfiles(	×
262	scalapb: ScalaPBSubsystem,
263	jdk: InternalJdk,
264	) -> ScalaPBShimCompiledClassfiles:
265	dest_dir = "classfiles"	×
266
267	scalapb_shim_content = read_resource(	×
268	"pants.backend.codegen.protobuf.scala", "ScalaPBShim.scala"
269	)
270	if not scalapb_shim_content:	×
271	raise AssertionError("Unable to find ScalaParser.scala resource.")	×
272
273	scalapb_shim_source = FileContent("ScalaPBShim.scala", scalapb_shim_content)	×
274
275	lockfile_request = GenerateJvmLockfileFromTool.create(scalapb)	×
276	scala_artifacts = await resolve_scala_artifacts_for_version(	×
277	ScalaArtifactsForVersionRequest(SHIM_SCALA_VERSION)
278	)
279	tool_classpath, shim_classpath, source_digest = await concurrently(	×
280	materialize_classpath_for_tool(
281	ToolClasspathRequest(
282	prefix="__toolcp",
283	artifact_requirements=ArtifactRequirements.from_coordinates(
284	scala_artifacts.all_coordinates
285	),
286	)
287	),
288	materialize_classpath_for_tool(
289	ToolClasspathRequest(prefix="__shimcp", lockfile=lockfile_request)
290	),
291	create_digest(CreateDigest([scalapb_shim_source, Directory(dest_dir)])),
292	)
293
294	merged_digest = await merge_digests(	×
295	MergeDigests((tool_classpath.digest, shim_classpath.digest, source_digest))
296	)
297
298	process_result = await fallible_to_exec_result_or_raise(	×
299	**implicitly(
300	JvmProcess(
301	jdk=jdk,
302	classpath_entries=tool_classpath.classpath_entries(),
303	argv=[
304	"scala.tools.nsc.Main",
305	"-bootclasspath",
306	":".join(tool_classpath.classpath_entries()),
307	"-classpath",
308	":".join(shim_classpath.classpath_entries()),
309	"-d",
310	dest_dir,
311	scalapb_shim_source.path,
312	],
313	input_digest=merged_digest,
314	extra_jvm_options=scalapb.jvm_options,
315	output_directories=(dest_dir,),
316	description="Compile ScalaPB shim with scalac",
317	level=LogLevel.DEBUG,
318	# NB: We do not use nailgun for this process, since it is launched exactly once.
319	use_nailgun=False,
320	)
321	)
322	)
323	stripped_classfiles_digest = await remove_prefix(	×
324	RemovePrefix(process_result.output_digest, dest_dir)
325	)
326	return ScalaPBShimCompiledClassfiles(digest=stripped_classfiles_digest)	×
327
328
UNCOV 329	def rules():	×
UNCOV 330	return [	×
331	*collect_rules(),
332	*lockfile.rules(),
333	*dependency_inference.rules(),
334	*symbol_mapper.rules(),
335	UnionRule(GenerateSourcesRequest, GenerateScalaFromProtobufRequest),
336	UnionRule(ExportableTool, ScalaPBSubsystem),
337	*protoc.rules(),
338	ProtobufSourceTarget.register_plugin_field(PrefixedJvmJdkField),
339	ProtobufSourcesGeneratorTarget.register_plugin_field(PrefixedJvmJdkField),
340	ProtobufSourceTarget.register_plugin_field(PrefixedJvmResolveField),
341	ProtobufSourcesGeneratorTarget.register_plugin_field(PrefixedJvmResolveField),
342	# Rules to avoid rule graph errors.
343	*artifact_mapper.rules(),
344	*distdir.rules(),
345	]

pantsbuild / pants / 20332790708

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous