25443604553

Committed 06 May 2026 03:05PM UTC coverage: 92.879% (-0.04%) from 92.915%

Build # 25443604553

Build Type

push

github

Committed by

web-flow

Commit Message

[pants_ng] Scaffolding for a pants_ng mode. (#23319)

In this mode the command line is parsed as an
NG invocation, and dispatched appropriately.

Of course at the moment there are no
implementations to dispatch to. That will follow.

This does expose a new option, `pants_ng` to users. 
There is a big warning not to set it, but we're not trying
to hide that we're working on a new thing, so I am
comfortable with this.

Coverage Stats

25 of 76 new or added lines in 9 files covered. (32.89%)

1294 existing lines in 76 files now uncovered.

92234 of 99306 relevant lines covered (92.88%)

4.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

99.14

/src/python/pants/backend/codegen/protobuf/scala/rules.py

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
UNCOV 3	from __future__ import annotations	1✔
4
UNCOV 5	import os	1✔
UNCOV 6	from dataclasses import dataclass	1✔
7
UNCOV 8	from pants.backend.codegen.protobuf import protoc	1✔
UNCOV 9	from pants.backend.codegen.protobuf.protoc import Protoc	1✔
UNCOV 10	from pants.backend.codegen.protobuf.scala import dependency_inference, symbol_mapper	1✔
UNCOV 11	from pants.backend.codegen.protobuf.scala.subsystem import PluginArtifactSpec, ScalaPBSubsystem	1✔
UNCOV 12	from pants.backend.codegen.protobuf.target_types import (	1✔
13	ProtobufSourceField,
14	ProtobufSourcesGeneratorTarget,
15	ProtobufSourceTarget,
16	)
UNCOV 17	from pants.backend.scala.target_types import ScalaSourceField	1✔
UNCOV 18	from pants.backend.scala.util_rules.versions import (	1✔
19	ScalaArtifactsForVersionRequest,
20	ScalaVersion,
21	resolve_scala_artifacts_for_version,
22	)
UNCOV 23	from pants.core.goals.resolves import ExportableTool	1✔
UNCOV 24	from pants.core.util_rules import distdir	1✔
UNCOV 25	from pants.core.util_rules.env_vars import environment_vars_subset	1✔
UNCOV 26	from pants.core.util_rules.external_tool import download_external_tool	1✔
UNCOV 27	from pants.core.util_rules.source_files import SourceFilesRequest	1✔
UNCOV 28	from pants.core.util_rules.stripped_source_files import strip_source_roots	1✔
UNCOV 29	from pants.engine.env_vars import EnvironmentVarsRequest	1✔
UNCOV 30	from pants.engine.fs import (	1✔
31	AddPrefix,
32	CreateDigest,
33	Digest,
34	Directory,
35	FileContent,
36	MergeDigests,
37	RemovePrefix,
38	)
UNCOV 39	from pants.engine.internals.graph import transitive_targets	1✔
UNCOV 40	from pants.engine.internals.native_engine import EMPTY_DIGEST	1✔
UNCOV 41	from pants.engine.internals.selectors import concurrently	1✔
UNCOV 42	from pants.engine.intrinsics import (	1✔
43	add_prefix,
44	create_digest,
45	digest_to_snapshot,
46	merge_digests,
47	remove_prefix,
48	)
UNCOV 49	from pants.engine.platform import Platform	1✔
UNCOV 50	from pants.engine.process import fallible_to_exec_result_or_raise	1✔
UNCOV 51	from pants.engine.rules import collect_rules, implicitly, rule	1✔
UNCOV 52	from pants.engine.target import GeneratedSources, GenerateSourcesRequest, TransitiveTargetsRequest	1✔
UNCOV 53	from pants.engine.unions import UnionRule	1✔
UNCOV 54	from pants.jvm.compile import ClasspathEntry	1✔
UNCOV 55	from pants.jvm.dependency_inference import artifact_mapper	1✔
UNCOV 56	from pants.jvm.goals import lockfile	1✔
UNCOV 57	from pants.jvm.jdk_rules import InternalJdk, JvmProcess	1✔
UNCOV 58	from pants.jvm.resolve.common import ArtifactRequirements, GatherJvmCoordinatesRequest	1✔
UNCOV 59	from pants.jvm.resolve.coursier_fetch import (	1✔
60	ToolClasspath,
61	ToolClasspathRequest,
62	materialize_classpath_for_tool,
63	)
UNCOV 64	from pants.jvm.resolve.jvm_tool import (	1✔
65	GenerateJvmLockfileFromTool,
66	gather_coordinates_for_jvm_lockfile,
67	)
UNCOV 68	from pants.jvm.target_types import PrefixedJvmJdkField, PrefixedJvmResolveField	1✔
UNCOV 69	from pants.source.source_root import SourceRootRequest, get_source_root	1✔
UNCOV 70	from pants.util.logging import LogLevel	1✔
UNCOV 71	from pants.util.ordered_set import FrozenOrderedSet	1✔
UNCOV 72	from pants.util.resources import read_resource	1✔
73
74
UNCOV 75	class GenerateScalaFromProtobufRequest(GenerateSourcesRequest):	1✔
UNCOV 76	input = ProtobufSourceField	1✔
UNCOV 77	output = ScalaSourceField	1✔
78
79
UNCOV 80	class ScalaPBShimCompiledClassfiles(ClasspathEntry):	1✔
UNCOV 81	pass	1✔
82
83
UNCOV 84	@dataclass(frozen=True)	1✔
UNCOV 85	class MaterializeJvmPluginRequest:	1✔
UNCOV 86	plugin: PluginArtifactSpec	1✔
87
88
UNCOV 89	@dataclass(frozen=True)	1✔
UNCOV 90	class MaterializedJvmPlugin:	1✔
UNCOV 91	name: str	1✔
UNCOV 92	classpath: ToolClasspath	1✔
93
UNCOV 94	def setup_arg(self, plugin_relpath: str) -> str:	1✔
UNCOV 95	classpath_arg = ":".join(self.classpath.classpath_entries(plugin_relpath))	1✔
UNCOV 96	return f"--jvm-plugin={self.name}={classpath_arg}"	1✔
97
98
UNCOV 99	@dataclass(frozen=True)	1✔
UNCOV 100	class MaterializeJvmPluginsRequest:	1✔
UNCOV 101	plugins: tuple[PluginArtifactSpec, ...]	1✔
102
103
UNCOV 104	@dataclass(frozen=True)	1✔
UNCOV 105	class MaterializedJvmPlugins:	1✔
UNCOV 106	digest: Digest	1✔
UNCOV 107	plugins: tuple[MaterializedJvmPlugin, ...]	1✔
108
UNCOV 109	def setup_args(self, plugins_relpath: str) -> tuple[str, ...]:	1✔
UNCOV 110	return tuple(p.setup_arg(os.path.join(plugins_relpath, p.name)) for p in self.plugins)	1✔
111
112
UNCOV 113	@rule	1✔
UNCOV 114	async def materialize_jvm_plugin(request: MaterializeJvmPluginRequest) -> MaterializedJvmPlugin:	1✔
UNCOV 115	requirements = await gather_coordinates_for_jvm_lockfile(	1✔
116	GatherJvmCoordinatesRequest(
117	artifact_inputs=FrozenOrderedSet([request.plugin.artifact]),
118	option_name="[scalapb].jvm_plugins",
119	)
120	)
UNCOV 121	classpath = await materialize_classpath_for_tool(	1✔
122	ToolClasspathRequest(artifact_requirements=requirements)
123	)
UNCOV 124	return MaterializedJvmPlugin(name=request.plugin.name, classpath=classpath)	1✔
125
126
UNCOV 127	@rule	1✔
UNCOV 128	async def materialize_jvm_plugins(	1✔
129	request: MaterializeJvmPluginsRequest,
130	) -> MaterializedJvmPlugins:
UNCOV 131	materialized_plugins = await concurrently(	1✔
132	materialize_jvm_plugin(MaterializeJvmPluginRequest(plugin)) for plugin in request.plugins
133	)
UNCOV 134	plugin_digests = await concurrently(	1✔
135	add_prefix(AddPrefix(p.classpath.digest, p.name)) for p in materialized_plugins
136	)
UNCOV 137	merged_plugins_digest = await merge_digests(MergeDigests(plugin_digests))	1✔
UNCOV 138	return MaterializedJvmPlugins(merged_plugins_digest, materialized_plugins)	1✔
139
140
UNCOV 141	@rule(desc="Generate Scala from Protobuf", level=LogLevel.DEBUG)	1✔
UNCOV 142	async def generate_scala_from_protobuf(	1✔
143	request: GenerateScalaFromProtobufRequest,
144	protoc: Protoc,
145	scalapb: ScalaPBSubsystem,
146	shim_classfiles: ScalaPBShimCompiledClassfiles,
147	jdk: InternalJdk,
148	platform: Platform,
149	) -> GeneratedSources:
UNCOV 150	output_dir = "_generated_files"	1✔
UNCOV 151	toolcp_relpath = "__toolcp"	1✔
UNCOV 152	shimcp_relpath = "__shimcp"	1✔
UNCOV 153	plugins_relpath = "__plugins"	1✔
UNCOV 154	protoc_relpath = "__protoc"	1✔
155
UNCOV 156	lockfile_request = GenerateJvmLockfileFromTool.create(scalapb)	1✔
UNCOV 157	(	1✔
158	downloaded_protoc_binary,
159	tool_classpath,
160	empty_output_dir,
161	transitive_targets_for_protobuf_source,
162	inherit_env,
163	) = await concurrently(
164	download_external_tool(protoc.get_request(platform)),
165	materialize_classpath_for_tool(ToolClasspathRequest(lockfile=lockfile_request)),
166	create_digest(CreateDigest([Directory(output_dir)])),
167	transitive_targets(
168	TransitiveTargetsRequest([request.protocol_target.address]), **implicitly()
169	),
170	# Need PATH so that ScalaPB can invoke `mkfifo`.
171	environment_vars_subset(EnvironmentVarsRequest(requested=["PATH"]), **implicitly()),
172	)
173
174	# NB: By stripping the source roots, we avoid having to set the value `--proto_path`
175	# for Protobuf imports to be discoverable.
UNCOV 176	all_sources_stripped, target_sources_stripped = await concurrently(	1✔
177	strip_source_roots(
178	**implicitly(
179	SourceFilesRequest(
180	tgt[ProtobufSourceField]
181	for tgt in transitive_targets_for_protobuf_source.closure
182	if tgt.has_field(ProtobufSourceField)
183	)
184	)
185	),
186	strip_source_roots(
187	**implicitly(SourceFilesRequest([request.protocol_target[ProtobufSourceField]]))
188	),
189	)
190
UNCOV 191	merged_jvm_plugins_digest = EMPTY_DIGEST	1✔
UNCOV 192	maybe_jvm_plugins_setup_args: tuple[str, ...] = ()	1✔
UNCOV 193	maybe_jvm_plugins_output_args: tuple[str, ...] = ()	1✔
UNCOV 194	jvm_plugins = scalapb.jvm_plugins	1✔
UNCOV 195	if jvm_plugins:	1✔
UNCOV 196	materialized_jvm_plugins = await materialize_jvm_plugins(	1✔
197	MaterializeJvmPluginsRequest(jvm_plugins)
198	)
UNCOV 199	merged_jvm_plugins_digest = materialized_jvm_plugins.digest	1✔
UNCOV 200	maybe_jvm_plugins_setup_args = materialized_jvm_plugins.setup_args(plugins_relpath)	1✔
UNCOV 201	maybe_jvm_plugins_output_args = tuple(	1✔
202	f"--{plugin.name}_out={output_dir}" for plugin in materialized_jvm_plugins.plugins
203	)
204
UNCOV 205	extra_immutable_input_digests = {	1✔
206	toolcp_relpath: tool_classpath.digest,
207	shimcp_relpath: shim_classfiles.digest,
208	plugins_relpath: merged_jvm_plugins_digest,
209	protoc_relpath: downloaded_protoc_binary.digest,
210	}
211
UNCOV 212	input_digest = await merge_digests(	1✔
213	MergeDigests([all_sources_stripped.snapshot.digest, empty_output_dir])
214	)
215
UNCOV 216	result = await fallible_to_exec_result_or_raise(	1✔
217	**implicitly(
218	JvmProcess(
219	jdk=jdk,
220	classpath_entries=[
221	*tool_classpath.classpath_entries(toolcp_relpath),
222	shimcp_relpath,
223	],
224	argv=[
225	"org.pantsbuild.backend.scala.scalapb.ScalaPBShim",
226	f"--protoc={os.path.join(protoc_relpath, downloaded_protoc_binary.exe)}",
227	*maybe_jvm_plugins_setup_args,
228	f"--scala_out={output_dir}",
229	*maybe_jvm_plugins_output_args,
230	*target_sources_stripped.snapshot.files,
231	],
232	input_digest=input_digest,
233	extra_immutable_input_digests=extra_immutable_input_digests,
234	extra_nailgun_keys=extra_immutable_input_digests,
235	description=f"Generating Scala sources from {request.protocol_target.address}.",
236	level=LogLevel.DEBUG,
237	output_directories=(output_dir,),
238	extra_env=inherit_env,
239	)
240	)
241	)
242
UNCOV 243	normalized_digest, source_root = await concurrently(	1✔
244	remove_prefix(RemovePrefix(result.output_digest, output_dir)),
245	get_source_root(SourceRootRequest.for_target(request.protocol_target)),
246	)
247
UNCOV 248	source_root_restored = (	1✔
249	await digest_to_snapshot(**implicitly(AddPrefix(normalized_digest, source_root.path)))
250	if source_root.path != "."
251	else await digest_to_snapshot(normalized_digest)
252	)
UNCOV 253	return GeneratedSources(source_root_restored)	1✔
254
255
UNCOV 256	SHIM_SCALA_VERSION = ScalaVersion.parse("2.13.7")	1✔
257
258
259	# TODO(13879): Consolidate compilation of wrapper binaries to common rules.
UNCOV 260	@rule	1✔
UNCOV 261	async def setup_scalapb_shim_classfiles(	1✔
262	scalapb: ScalaPBSubsystem,
263	jdk: InternalJdk,
264	) -> ScalaPBShimCompiledClassfiles:
UNCOV 265	dest_dir = "classfiles"	1✔
266
UNCOV 267	scalapb_shim_content = read_resource(	1✔
268	"pants.backend.codegen.protobuf.scala", "ScalaPBShim.scala"
269	)
UNCOV 270	if not scalapb_shim_content:	1✔
271	raise AssertionError("Unable to find ScalaParser.scala resource.")	×
272
UNCOV 273	scalapb_shim_source = FileContent("ScalaPBShim.scala", scalapb_shim_content)	1✔
274
UNCOV 275	lockfile_request = GenerateJvmLockfileFromTool.create(scalapb)	1✔
UNCOV 276	scala_artifacts = await resolve_scala_artifacts_for_version(	1✔
277	ScalaArtifactsForVersionRequest(SHIM_SCALA_VERSION)
278	)
UNCOV 279	tool_classpath, shim_classpath, source_digest = await concurrently(	1✔
280	materialize_classpath_for_tool(
281	ToolClasspathRequest(
282	prefix="__toolcp",
283	artifact_requirements=ArtifactRequirements.from_coordinates(
284	scala_artifacts.all_coordinates
285	),
286	)
287	),
288	materialize_classpath_for_tool(
289	ToolClasspathRequest(prefix="__shimcp", lockfile=lockfile_request)
290	),
291	create_digest(CreateDigest([scalapb_shim_source, Directory(dest_dir)])),
292	)
293
UNCOV 294	merged_digest = await merge_digests(	1✔
295	MergeDigests((tool_classpath.digest, shim_classpath.digest, source_digest))
296	)
297
UNCOV 298	process_result = await fallible_to_exec_result_or_raise(	1✔
299	**implicitly(
300	JvmProcess(
301	jdk=jdk,
302	classpath_entries=tool_classpath.classpath_entries(),
303	argv=[
304	"scala.tools.nsc.Main",
305	"-bootclasspath",
306	":".join(tool_classpath.classpath_entries()),
307	"-classpath",
308	":".join(shim_classpath.classpath_entries()),
309	"-d",
310	dest_dir,
311	scalapb_shim_source.path,
312	],
313	input_digest=merged_digest,
314	extra_jvm_options=scalapb.jvm_options,
315	output_directories=(dest_dir,),
316	description="Compile ScalaPB shim with scalac",
317	level=LogLevel.DEBUG,
318	# NB: We do not use nailgun for this process, since it is launched exactly once.
319	use_nailgun=False,
320	)
321	)
322	)
UNCOV 323	stripped_classfiles_digest = await remove_prefix(	1✔
324	RemovePrefix(process_result.output_digest, dest_dir)
325	)
UNCOV 326	return ScalaPBShimCompiledClassfiles(digest=stripped_classfiles_digest)	1✔
327
328
UNCOV 329	def rules():	1✔
UNCOV 330	return [	1✔
331	*collect_rules(),
332	*lockfile.rules(),
333	*dependency_inference.rules(),
334	*symbol_mapper.rules(),
335	UnionRule(GenerateSourcesRequest, GenerateScalaFromProtobufRequest),
336	UnionRule(ExportableTool, ScalaPBSubsystem),
337	*protoc.rules(),
338	ProtobufSourceTarget.register_plugin_field(PrefixedJvmJdkField),
339	ProtobufSourcesGeneratorTarget.register_plugin_field(PrefixedJvmJdkField),
340	ProtobufSourceTarget.register_plugin_field(PrefixedJvmResolveField),
341	ProtobufSourcesGeneratorTarget.register_plugin_field(PrefixedJvmResolveField),
342	# Rules to avoid rule graph errors.
343	*artifact_mapper.rules(),
344	*distdir.rules(),
345	]

pantsbuild / pants / 25443604553

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous