19015773527

Committed 02 Nov 2025 05:33PM UTC coverage: 17.872% (-62.4%) from 80.3%

Build # 19015773527

Build Type

Pull #22816

github

Committed by

web-flow

Commit Message

Merge a12d75757 into 6c024e162

Pull Request Pull Request #22816: Update Pants internal Python to 3.14

Run Details

4 of 5 new or added lines in 3 files covered. (80.0%)

28452 existing lines in 683 files now uncovered.

9831 of 55007 relevant lines covered (17.87%)

0.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/python/pants/jvm/resolve/coursier_setup.py

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
UNCOV 4	from __future__ import annotations	×
5
UNCOV 6	import os	×
UNCOV 7	import shlex	×
UNCOV 8	import textwrap	×
UNCOV 9	from collections.abc import Iterable	×
UNCOV 10	from dataclasses import dataclass	×
UNCOV 11	from hashlib import sha256	×
UNCOV 12	from typing import ClassVar	×
13
UNCOV 14	from pants.core.goals.resolves import ExportableTool	×
UNCOV 15	from pants.core.util_rules import external_tool	×
UNCOV 16	from pants.core.util_rules.adhoc_binaries import PythonBuildStandaloneBinary	×
UNCOV 17	from pants.core.util_rules.external_tool import (	×
18	DownloadedExternalTool,
19	TemplatedExternalTool,
20	download_external_tool,
21	)
UNCOV 22	from pants.core.util_rules.system_binaries import BashBinary, MkdirBinary	×
UNCOV 23	from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests	×
UNCOV 24	from pants.engine.intrinsics import create_digest, merge_digests	×
UNCOV 25	from pants.engine.platform import Platform	×
UNCOV 26	from pants.engine.process import Process	×
UNCOV 27	from pants.engine.rules import collect_rules, concurrently, rule	×
UNCOV 28	from pants.engine.unions import UnionRule	×
UNCOV 29	from pants.option.option_types import StrListOption, StrOption	×
UNCOV 30	from pants.util.frozendict import FrozenDict	×
UNCOV 31	from pants.util.logging import LogLevel	×
UNCOV 32	from pants.util.memo import memoized_property	×
UNCOV 33	from pants.util.ordered_set import FrozenOrderedSet	×
UNCOV 34	from pants.util.strutil import softwrap	×
35
UNCOV 36	COURSIER_POST_PROCESSING_SCRIPT = textwrap.dedent( # noqa: PNT20	×
37	"""\
38	import json
39	import sys
40	import os
41	from pathlib import PurePath
42	from shutil import copyfile
43
44	report = json.load(open(sys.argv[1]))
45
46	# Mapping from dest path to source path. It is ok to capture the same output filename multiple
47	# times if the source is the same as well.
48	classpath = dict()
49	for dep in report['dependencies']:
50	if not dep.get('file'):
51	raise Exception(
52	f"No jar found for {dep['coord']}. Check that it's available in the"
53	+ " repositories configured in [coursier].repos in pants.toml."
54	)
55	source = PurePath(dep['file'])
56	dest_name = dep['coord'].replace(":", "_")
57	_, ext = os.path.splitext(source)
58	classpath_dest = f"classpath/{dest_name}{ext}"
59
60	existing_source = classpath.get(classpath_dest)
61	if existing_source:
62	if existing_source == source:
63	# We've already captured this file.
64	continue
65	raise Exception(
66	f"Duplicate jar name {classpath_dest} with incompatible source:\\n"
67	f" {source}\\n"
68	f" {existing_source}\\n"
69	)
70	classpath[classpath_dest] = source
71	copyfile(source, classpath_dest)
72	"""
73	)
74
UNCOV 75	COURSIER_FETCH_WRAPPER_SCRIPT = textwrap.dedent( # noqa: PNT20	×
76	"""\
77	set -eux
78
79	coursier_exe="$1"
80	shift
81	json_output_file="$1"
82	shift
83
84	working_dir="$(pwd)"
85	"$coursier_exe" fetch {repos_args} \
86	--json-output-file="$json_output_file" \
87	"${{@//{coursier_working_directory}/$working_dir}}"
88	{mkdir} -p classpath
89	{python_path} {coursier_bin_dir}/coursier_post_processing_script.py "$json_output_file"
90	"""
91	)
92
93	# TODO: Coursier renders setrlimit error line on macOS.
94	# see https://github.com/pantsbuild/pants/issues/13942.
UNCOV 95	POST_PROCESS_COURSIER_STDERR_SCRIPT = textwrap.dedent( # noqa: PNT20	×
96	"""\
97	#!{python_path}
98	import sys
99	from subprocess import run, PIPE
100
101	proc = run(sys.argv[1:], stdout=PIPE, stderr=PIPE)
102
103	sys.stdout.buffer.write(proc.stdout)
104	sys.stderr.buffer.write(proc.stderr.replace(b"setrlimit to increase file descriptor limit failed, errno 22\\n", b""))
105	sys.exit(proc.returncode)
106	"""
107	)
108
109
UNCOV 110	class CoursierSubsystem(TemplatedExternalTool):	×
UNCOV 111	options_scope = "coursier"	×
UNCOV 112	name = "coursier"	×
UNCOV 113	help = "A dependency resolver for the Maven ecosystem. (https://get-coursier.io/)"	×
114
UNCOV 115	default_version = "v2.1.6"	×
UNCOV 116	default_known_versions = [	×
117	"v2.1.6\|macos_arm64 \|746b3e346fa2c0107fdbc8a627890d495cb09dee4f8dcc87146bdb45941088cf\|20829782\|https://github.com/VirtusLab/coursier-m1/releases/download/v2.1.6/cs-aarch64-apple-darwin.gz",
118	"v2.1.6\|linux_arm64 \|33330ca433781c9db9458e15d2d32e5d795de3437771647e26835e8b1391af82\|20899290\|https://github.com/VirtusLab/coursier-m1/releases/download/v2.1.6/cs-aarch64-pc-linux.gz",
119	"v2.1.6\|linux_x86_64\|af7234f8802107f5e1130307ef8a5cc90262d392f16ddff7dce27a4ed0ddd292\|20681688",
120	"v2.1.6\|macos_x86_64\|36a5d42a0724be2ac39d0ebd8869b985e3d58ceb121bc60389ee2d6d7408dd56\|20037412",
121	"v2.1.0-M5-18-gfebf9838c\|linux_arm64 \|d4ad15ba711228041ad8a46d848c83c8fbc421d7b01c415d8022074dd609760f\|19264005",
122	"v2.1.0-M5-18-gfebf9838c\|linux_x86_64\|3e1a1ad1010d5582e9e43c5a26b273b0147baee5ebd27d3ac1ab61964041c90b\|19551533",
123	"v2.1.0-M5-18-gfebf9838c\|macos_arm64 \|d13812c5a5ef4c9b3e25cc046d18addd09bacd149f95b20a14e4d2a73e358ecf\|18826510",
124	"v2.1.0-M5-18-gfebf9838c\|macos_x86_64\|d13812c5a5ef4c9b3e25cc046d18addd09bacd149f95b20a14e4d2a73e358ecf\|18826510",
125	"v2.0.16-169-g194ebc55c\|linux_arm64 \|da38c97d55967505b8454c20a90370c518044829398b9bce8b637d194d79abb3\|18114472",
126	"v2.0.16-169-g194ebc55c\|linux_x86_64\|4c61a634c4bd2773b4543fe0fc32210afd343692891121cddb447204b48672e8\|18486946",
127	"v2.0.16-169-g194ebc55c\|macos_arm64 \|15bce235d223ef1d022da30b67b4c64e9228d236b876c834b64e029bbe824c6f\|17957182",
128	"v2.0.16-169-g194ebc55c\|macos_x86_64\|15bce235d223ef1d022da30b67b4c64e9228d236b876c834b64e029bbe824c6f\|17957182",
129	]
UNCOV 130	default_url_template = (	×
131	"https://github.com/coursier/coursier/releases/download/{version}/cs-{platform}.gz"
132	)
UNCOV 133	default_url_platform_mapping = {	×
134	# By default we pull x86 binaries for Mac, since arm binaries
135	# are unavailable for older supported versions of coursier. They work fine with rosetta.
136	# For recent versions, arm binaries for mac and linux are available
137	# at https://github.com/VirtusLab/coursier-m1/
138	# Set the fifth field in known_versions to pull from this alternative source.
139	"macos_arm64": "x86_64-apple-darwin",
140	"macos_x86_64": "x86_64-apple-darwin",
141	"linux_arm64": "aarch64-pc-linux",
142	"linux_x86_64": "x86_64-pc-linux",
143	}
144
UNCOV 145	repos = StrListOption(	×
146	default=[
147	"https://maven-central.storage-download.googleapis.com/maven2",
148	"https://repo1.maven.org/maven2",
149	],
150	help=softwrap(
151	"""
152	Maven style repositories to resolve artifacts from.
153
154	Coursier will resolve these repositories in the order in which they are
155	specified, and re-ordering repositories will cause artifacts to be
156	re-downloaded. This can result in artifacts in lockfiles becoming invalid.
157	"""
158	),
159	)
160
UNCOV 161	jvm_index = StrOption(	×
162	default="",
163	help=softwrap(
164	"""
165	The JVM index to be used by Coursier.
166
167	Possible values are:
168	- cs: The default JVM index used and maintained by Coursier.
169	- cs-maven: Fetches a JVM index from the io.get-coursier:jvm-index Maven repository.
170	- <URL>: An arbitrary URL for a JVM index. Ex. https://url/of/your/index.json
171	"""
172	),
173	)
174
UNCOV 175	def generate_exe(self, plat: Platform) -> str:	×
176	tool_version = self.known_version(plat)	×
177	url = (tool_version and tool_version.url_override) or self.generate_url(plat)	×
178	archive_filename = os.path.basename(url)	×
179	filename = os.path.splitext(archive_filename)[0]	×
180	return f"./{filename}"	×
181
182
UNCOV 183	@dataclass(frozen=True)	×
UNCOV 184	class Coursier:	×
185	"""The Coursier tool and various utilities, prepared for use via `immutable_input_digests`."""
186
UNCOV 187	coursier: DownloadedExternalTool	×
UNCOV 188	_digest: Digest	×
UNCOV 189	repos: FrozenOrderedSet[str]	×
UNCOV 190	jvm_index: str	×
UNCOV 191	_append_only_caches: FrozenDict[str, str]	×
192
UNCOV 193	bin_dir: ClassVar[str] = "__coursier"	×
UNCOV 194	fetch_wrapper_script: ClassVar[str] = f"{bin_dir}/coursier_fetch_wrapper_script.sh"	×
UNCOV 195	post_processing_script: ClassVar[str] = f"{bin_dir}/coursier_post_processing_script.py"	×
UNCOV 196	post_process_stderr: ClassVar[str] = f"{bin_dir}/coursier_post_process_stderr.py"	×
UNCOV 197	cache_name: ClassVar[str] = "coursier"	×
UNCOV 198	cache_dir: ClassVar[str] = ".cache"	×
UNCOV 199	working_directory_placeholder: ClassVar[str] = "___COURSIER_WORKING_DIRECTORY___"	×
200
UNCOV 201	def args(self, args: Iterable[str], *, wrapper: Iterable[str] = ()) -> tuple[str, ...]:	×
202	return (	×
203	self.post_process_stderr,
204	*wrapper,
205	os.path.join(self.bin_dir, self.coursier.exe),
206	*args,
207	)
208
UNCOV 209	@memoized_property	×
UNCOV 210	def _coursier_cache_prefix(self) -> str:	×
211	"""Returns a key for `COURSIER_CACHE` determined by the configured repositories.
212
213	This helps us avoid a cache poisoning issue that we uncovered in #14577.
214	"""
215	sha = sha256()	×
216	for repo in self.repos:	×
217	sha.update(repo.encode("utf-8"))	×
218	return sha.digest().hex()	×
219
UNCOV 220	@property	×
UNCOV 221	def env(self) -> dict[str, str]:	×
222	# NB: These variables have changed a few times, and they change again on `main`. But as of
223	# `v2.0.16+73-gddc6d9cc9` they are accurate. See:
224	# https://github.com/coursier/coursier/blob/v2.0.16+73-gddc6d9cc9/modules/paths/src/main/java/coursier/paths/CoursierPaths.java#L38-L48
225	return {	×
226	# Maven artifacts and JDK tarballs go here
227	"COURSIER_CACHE": f"{self.cache_dir}/{self._coursier_cache_prefix}/jdk",
228	# extracted JDK tarballs go here
229	"COURSIER_ARCHIVE_CACHE": f"{self.cache_dir}/arc",
230	"COURSIER_JVM_CACHE": f"{self.cache_dir}/v1",
231	}
232
UNCOV 233	@property	×
UNCOV 234	def append_only_caches(self) -> dict[str, str]:	×
235	return {self.cache_name: self.cache_dir, **self._append_only_caches}	×
236
UNCOV 237	@property	×
UNCOV 238	def immutable_input_digests(self) -> dict[str, Digest]:	×
239	return {self.bin_dir: self._digest}	×
240
241
UNCOV 242	@dataclass(frozen=True)	×
UNCOV 243	class CoursierFetchProcess:	×
UNCOV 244	args: tuple[str, ...]	×
UNCOV 245	input_digest: Digest	×
UNCOV 246	output_directories: tuple[str, ...]	×
UNCOV 247	output_files: tuple[str, ...]	×
UNCOV 248	description: str	×
249
250
UNCOV 251	@rule	×
UNCOV 252	async def invoke_coursier_wrapper(	×
253	bash: BashBinary,
254	coursier: Coursier,
255	request: CoursierFetchProcess,
256	) -> Process:
257	return Process(	×
258	argv=coursier.args(
259	request.args,
260	wrapper=[bash.path, coursier.fetch_wrapper_script],
261	),
262	input_digest=request.input_digest,
263	immutable_input_digests=coursier.immutable_input_digests,
264	output_directories=request.output_directories,
265	output_files=request.output_files,
266	append_only_caches=coursier.append_only_caches,
267	env=coursier.env,
268	description=request.description,
269	level=LogLevel.DEBUG,
270	)
271
272
UNCOV 273	@rule	×
UNCOV 274	async def setup_coursier(	×
275	coursier_subsystem: CoursierSubsystem,
276	python: PythonBuildStandaloneBinary,
277	platform: Platform,
278	mkdir: MkdirBinary,
279	) -> Coursier:
280	repos_args = (	×
281	" ".join(f"-r={shlex.quote(repo)}" for repo in coursier_subsystem.repos) + " --no-default"
282	)
283	coursier_wrapper_script = COURSIER_FETCH_WRAPPER_SCRIPT.format(	×
284	repos_args=repos_args,
285	coursier_working_directory=Coursier.working_directory_placeholder,
286	python_path=shlex.quote(python.path),
287	coursier_bin_dir=shlex.quote(Coursier.bin_dir),
288	mkdir=shlex.quote(mkdir.path),
289	)
290
291	post_process_stderr = POST_PROCESS_COURSIER_STDERR_SCRIPT.format(python_path=python.path)	×
292
293	downloaded_coursier_get = download_external_tool(coursier_subsystem.get_request(platform))	×
294	wrapper_scripts_digest_get = create_digest(	×
295	CreateDigest(
296	[
297	FileContent(
298	os.path.basename(Coursier.fetch_wrapper_script),
299	coursier_wrapper_script.encode("utf-8"),
300	is_executable=True,
301	),
302	FileContent(
303	os.path.basename(Coursier.post_processing_script),
304	COURSIER_POST_PROCESSING_SCRIPT.encode("utf-8"),
305	is_executable=True,
306	),
307	FileContent(
308	os.path.basename(Coursier.post_process_stderr),
309	post_process_stderr.encode("utf-8"),
310	is_executable=True,
311	),
312	]
313	)
314	)
315
316	downloaded_coursier, wrapper_scripts_digest = await concurrently(	×
317	downloaded_coursier_get, wrapper_scripts_digest_get
318	)
319
320	return Coursier(	×
321	coursier=downloaded_coursier,
322	_digest=await merge_digests(
323	MergeDigests(
324	[
325	downloaded_coursier.digest,
326	wrapper_scripts_digest,
327	]
328	)
329	),
330	repos=FrozenOrderedSet(coursier_subsystem.repos),
331	jvm_index=coursier_subsystem.jvm_index,
332	_append_only_caches=python.APPEND_ONLY_CACHES,
333	)
334
335
UNCOV 336	def rules():	×
UNCOV 337	return [	×
338	*collect_rules(),
339	*external_tool.rules(),
340	UnionRule(ExportableTool, CoursierSubsystem),
341	]

pantsbuild / pants / 19015773527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous