20147226056

Committed 11 Dec 2025 08:58PM UTC coverage: 78.827% (-1.5%) from 80.293%

Build # 20147226056

Build Type

push

github

Committed by

web-flow

Commit Message

Forwarded the `style` and `complete-platform` args from pants.toml to PEX (#22910)

## Context

After Apple switched to the `arm64` architecture, some package
publishers stopped releasing `x86_64` variants of their packages for
`darwin`. As a result, generating a universal lockfile now fails because
no single package version is compatible with both `x86_64` and `arm64`
on `darwin`.

The solution is to use the `--style` and `--complete-platform` flags
with PEX. For example:
```
pex3 lock create \
    --style strict \
    --complete-platform 3rdparty/platforms/manylinux_2_28_aarch64.json \
    --complete-platform 3rdparty/platforms/macosx_26_0_arm64.json \
    -r 3rdparty/python/requirements_pyarrow.txt \
    -o python-pyarrow.lock
```

See the Slack discussion here:
https://pantsbuild.slack.com/archives/C046T6T9U/p1760098582461759

## Reproduction

* `BUILD`
```
python_requirement(
    name="awswrangler",
    requirements=["awswrangler==3.12.1"],
    resolve="awswrangler",
)
```
* Run `pants generate-lockfiles --resolve=awswrangler` on macOS with an
`arm64` CPU
```
pip: ERROR: Cannot install awswrangler==3.12.1 because these package versions have conflicting dependencies.
pip: ERROR: ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/topics/dependency-resolution/#dealing-with-dependency-conflicts
pip:  
pip:  The conflict is caused by:
pip:      awswrangler 3.12.1 depends on pyarrow<18.0.0 and >=8.0.0; sys_platform == "darwin" and platform_machine == "x86_64"
pip:      awswrangler 3.12.1 depends on pyarrow<21.0.0 and >=18.0.0; sys_platform != "darwin" or platform_machine != "x86_64"
pip:  
pip:  Additionally, some packages in these conflicts have no matching distributions available for your environment:
pip:      pyarrow
pip:  
pip:  To fix this you could try to:
pip:  1. loosen the range of package versions you've specified
pip:  2. remove package versions to allow pip to attempt to solve the dependency conflict
```

## Implementation
... (continued)

Run Details

77 of 100 new or added lines in 6 files covered. (77.0%)

868 existing lines in 42 files now uncovered.

74471 of 94474 relevant lines covered (78.83%)

3.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

60.19

/src/python/pants/jvm/jar_tool/jar_tool.py

# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import importlib.resources
import os
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from enum import Enum, unique

from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
from pants.core.goals.resolves import ExportableTool
from pants.engine.fs import (
    CreateDigest,
    Digest,
    DigestSubset,
    Directory,
    FileContent,
    FileEntry,
    MergeDigests,
    PathGlobs,
    RemovePrefix,
)
from pants.engine.intrinsics import (
    create_digest,
    digest_subset_to_digest,
    get_digest_entries,
    merge_digests,
    remove_prefix,
)
from pants.engine.process import execute_process_or_raise
from pants.engine.rules import collect_rules, concurrently, implicitly, rule
from pants.engine.unions import UnionRule
from pants.jvm.jdk_rules import InternalJdk, JvmProcess
from pants.jvm.jdk_rules import rules as jdk_rules
from pants.jvm.resolve.coursier_fetch import ToolClasspathRequest, materialize_classpath_for_tool
from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules
from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool, JvmToolBase
from pants.jvm.resolve.jvm_tool import rules as jvm_tool_rules
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel


class JarTool(JvmToolBase):
    options_scope = "jar_tool"
    help = "The Java Archive Tool"

    default_artifacts = (
        "args4j:args4j:2.33",
        "com.google.code.findbugs:jsr305:3.0.2",
        "com.google.guava:guava:18.0",
    )
    default_lockfile_resource = (
        "pants.jvm.jar_tool",
        "jar_tool.lock",
    )


@unique
class JarDuplicateAction(Enum):
    SKIP = "skip"
    REPLACE = "replace"
    CONCAT = "concat"
    CONCAT_TEXT = "concat_text"
    THROW = "throw"


@dataclass(frozen=True)
class JarToolRequest:
    jar_name: str
    digest: Digest
    main_class: str | None
    classpath_entries: tuple[str, ...]
    manifest: str | None
    jars: tuple[str, ...]
    file_mappings: FrozenDict[str, str]
    default_action: JarDuplicateAction | None
    policies: tuple[tuple[str, JarDuplicateAction], ...]
    skip: tuple[str, ...]
    compress: bool
    update: bool

    def __init__(
        self,
        *,
        jar_name: str,
        digest: Digest,
        main_class: str | None = None,
        classpath_entries: Iterable[str] | None = None,
        manifest: str | None = None,
        jars: Iterable[str] | None = None,
        file_mappings: Mapping[str, str] | None = None,
        files: Iterable[str] | None = None,
        default_action: JarDuplicateAction | None = None,
        policies: Iterable[tuple[str, str | JarDuplicateAction]] | None = None,
        skip: Iterable[str] | None = None,
        compress: bool = False,
        update: bool = False,
    ) -> None:
        _file_mappings = {**(file_mappings or {}), **({f: f for f in (files or [])})}

        object.__setattr__(self, "jar_name", jar_name)
        object.__setattr__(self, "digest", digest)
        object.__setattr__(self, "main_class", main_class)
        object.__setattr__(self, "manifest", manifest)
        object.__setattr__(self, "classpath_entries", tuple(classpath_entries or ()))
        object.__setattr__(self, "jars", tuple(jars or ()))
        object.__setattr__(self, "file_mappings", FrozenDict(_file_mappings))
        object.__setattr__(self, "default_action", default_action)
        object.__setattr__(self, "policies", tuple(JarToolRequest.__parse_policies(policies or ())))
        object.__setattr__(self, "skip", tuple(skip or ()))
        object.__setattr__(self, "compress", compress)
        object.__setattr__(self, "update", update)

    @staticmethod
    def __parse_policies(
        policies: Iterable[tuple[str, str | JarDuplicateAction]],
    ) -> Iterable[tuple[str, JarDuplicateAction]]:
        return [
            (
                pattern,
                (
                    action
                    if isinstance(action, JarDuplicateAction)
                    else JarDuplicateAction(action.lower())
                ),
            )
            for (pattern, action) in policies
        ]


_JAR_TOOL_MAIN_CLASS = "org.pantsbuild.tools.jar.Main"


@dataclass(frozen=True)
class JarToolCompiledClassfiles:
    digest: Digest


@rule
async def run_jar_tool(
    request: JarToolRequest, jdk: InternalJdk, tool: JarTool, jar_tool: JarToolCompiledClassfiles
) -> Digest:
    output_prefix = "__out"
    output_jarname = os.path.join(output_prefix, request.jar_name)

    tool_classpath, empty_output_digest = await concurrently(
        materialize_classpath_for_tool(
            ToolClasspathRequest(lockfile=GenerateJvmLockfileFromTool.create(tool))
        ),
        create_digest(CreateDigest([Directory(output_prefix)])),
    )

    toolcp_prefix = "__toolcp"
    jartoolcp_prefix = "__jartoolcp"
    input_prefix = "__in"
    immutable_input_digests = {
        toolcp_prefix: tool_classpath.digest,
        jartoolcp_prefix: jar_tool.digest,
        input_prefix: request.digest,
    }

    policies = ",".join(
        f"{pattern}={action.value.upper()}" for (pattern, action) in request.policies
    )
    file_mappings = ",".join(
        f"{os.path.join(input_prefix, fs_path)}={jar_path}"
        for fs_path, jar_path in request.file_mappings.items()
    )

    tool_process = JvmProcess(
        jdk=jdk,
        argv=[
            _JAR_TOOL_MAIN_CLASS,
            output_jarname,
            *((f"-main={request.main_class}",) if request.main_class else ()),
            *(
                (f"-classpath={','.join(request.classpath_entries)}",)
                if request.classpath_entries
                else ()
            ),
            *(
                (f"-manifest={os.path.join(input_prefix, request.manifest)}",)
                if request.manifest
                else ()
            ),
            *(
                (f"-jars={','.join([os.path.join(input_prefix, jar) for jar in request.jars])}",)
                if request.jars
                else ()
            ),
            *((f"-files={file_mappings}",) if file_mappings else ()),
            *(
                (f"-default_action={request.default_action.value.upper()}",)
                if request.default_action
                else ()
            ),
            *((f"-policies={policies}",) if policies else ()),
            *((f"-skip={','.join(request.skip)}",) if request.skip else ()),
            *(("-compress",) if request.compress else ()),
            *(("-update",) if request.update else ()),
        ],
        classpath_entries=[*tool_classpath.classpath_entries(toolcp_prefix), jartoolcp_prefix],
        input_digest=empty_output_digest,
        extra_immutable_input_digests=immutable_input_digests,
        extra_nailgun_keys=immutable_input_digests.keys(),
        description=f"Building jar {request.jar_name}",
        output_directories=(output_prefix,),
        level=LogLevel.DEBUG,
    )

    result = await execute_process_or_raise(**implicitly({tool_process: JvmProcess}))
    return await remove_prefix(RemovePrefix(result.output_digest, output_prefix))


_JAR_TOOL_SRC_PACKAGES = ["args4j", "jar_tool_source"]


def _load_jar_tool_sources() -> list[FileContent]:
    parent_module = ".".join(__name__.split(".")[:-1])
    result = []
    for package in _JAR_TOOL_SRC_PACKAGES:
        # pkg_path = package.replace(".", os.path.sep)
        # relative_folder = os.path.join("src", pkg_path)
        for resource in importlib.resources.files(parent_module).joinpath(package).iterdir():
            if not resource.is_file():
                continue
            result.append(
                FileContent(
                    path=os.path.join(package, resource.name),
                    content=resource.read_bytes(),
                )
            )
    return result


# TODO(13879): Consolidate compilation of wrapper binaries to common rules.
@rule
async def build_jar_tool(jdk: InternalJdk, tool: JarTool) -> JarToolCompiledClassfiles:
    source_digest = await create_digest(CreateDigest(_load_jar_tool_sources()))

    dest_dir = "classfiles"
    materialized_classpath, java_subset_digest, empty_dest_dir = await concurrently(
        materialize_classpath_for_tool(
            ToolClasspathRequest(
                prefix="__toolcp", lockfile=GenerateJvmLockfileFromTool.create(tool)
            )
        ),
        digest_subset_to_digest(
            DigestSubset(
                source_digest,
                PathGlobs(
                    ["**/*.java"],
                    glob_match_error_behavior=GlobMatchErrorBehavior.error,
                    description_of_origin="jar tool sources",
                ),
            )
        ),
        create_digest(CreateDigest([Directory(path=dest_dir)])),
    )

    merged_digest, src_entries = await concurrently(
        merge_digests(MergeDigests([materialized_classpath.digest, source_digest, empty_dest_dir])),
        get_digest_entries(java_subset_digest),
    )

    compile_result = await execute_process_or_raise(
        **implicitly(
            JvmProcess(
                jdk=jdk,
                classpath_entries=[f"{jdk.java_home}/lib/tools.jar"],
                argv=[
                    "com.sun.tools.javac.Main",
                    "-cp",
                    ":".join(materialized_classpath.classpath_entries()),
                    "-d",
                    dest_dir,
                    *[entry.path for entry in src_entries if isinstance(entry, FileEntry)],
                ],
                input_digest=merged_digest,
                output_directories=(dest_dir,),
                description="Compile jar-tool sources using javac.",
                level=LogLevel.DEBUG,
                use_nailgun=False,
            )
        )
    )

    stripped_classfiles_digest = await remove_prefix(
        RemovePrefix(compile_result.output_digest, dest_dir)
    )
    return JarToolCompiledClassfiles(digest=stripped_classfiles_digest)


def rules():
    return [
        *collect_rules(),
        *coursier_fetch_rules(),
        *jdk_rules(),
        *jvm_tool_rules(),
        UnionRule(ExportableTool, JarTool),
    ]

1	# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	4✔
5
6	import importlib.resources	4✔
7	import os	4✔
8	from collections.abc import Iterable, Mapping	4✔
9	from dataclasses import dataclass	4✔
10	from enum import Enum, unique	4✔
11
12	from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior	4✔
13	from pants.core.goals.resolves import ExportableTool	4✔
14	from pants.engine.fs import (	4✔
15	CreateDigest,
16	Digest,
17	DigestSubset,
18	Directory,
19	FileContent,
20	FileEntry,
21	MergeDigests,
22	PathGlobs,
23	RemovePrefix,
24	)
25	from pants.engine.intrinsics import (	4✔
26	create_digest,
27	digest_subset_to_digest,
28	get_digest_entries,
29	merge_digests,
30	remove_prefix,
31	)
32	from pants.engine.process import execute_process_or_raise	4✔
33	from pants.engine.rules import collect_rules, concurrently, implicitly, rule	4✔
34	from pants.engine.unions import UnionRule	4✔
35	from pants.jvm.jdk_rules import InternalJdk, JvmProcess	4✔
36	from pants.jvm.jdk_rules import rules as jdk_rules	4✔
37	from pants.jvm.resolve.coursier_fetch import ToolClasspathRequest, materialize_classpath_for_tool	4✔
38	from pants.jvm.resolve.coursier_fetch import rules as coursier_fetch_rules	4✔
39	from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool, JvmToolBase	4✔
40	from pants.jvm.resolve.jvm_tool import rules as jvm_tool_rules	4✔
41	from pants.util.frozendict import FrozenDict	4✔
42	from pants.util.logging import LogLevel	4✔
43
44
45	class JarTool(JvmToolBase):	4✔
46	options_scope = "jar_tool"	4✔
47	help = "The Java Archive Tool"	4✔
48
49	default_artifacts = (	4✔
50	"args4j:args4j:2.33",
51	"com.google.code.findbugs:jsr305:3.0.2",
52	"com.google.guava:guava:18.0",
53	)
54	default_lockfile_resource = (	4✔
55	"pants.jvm.jar_tool",
56	"jar_tool.lock",
57	)
58
59
60	@unique	4✔
61	class JarDuplicateAction(Enum):	4✔
62	SKIP = "skip"	4✔
63	REPLACE = "replace"	4✔
64	CONCAT = "concat"	4✔
65	CONCAT_TEXT = "concat_text"	4✔
66	THROW = "throw"	4✔
67
68
69	@dataclass(frozen=True)	4✔
70	class JarToolRequest:	4✔
71	jar_name: str	4✔
72	digest: Digest	4✔
73	main_class: str \| None	4✔
74	classpath_entries: tuple[str, ...]	4✔
75	manifest: str \| None	4✔
76	jars: tuple[str, ...]	4✔
77	file_mappings: FrozenDict[str, str]	4✔
78	default_action: JarDuplicateAction \| None	4✔
79	policies: tuple[tuple[str, JarDuplicateAction], ...]	4✔
80	skip: tuple[str, ...]	4✔
81	compress: bool	4✔
82	update: bool	4✔
83
84	def __init__(	4✔
85	self,
86	*,
87	jar_name: str,
88	digest: Digest,
89	main_class: str \| None = None,
90	classpath_entries: Iterable[str] \| None = None,
91	manifest: str \| None = None,
92	jars: Iterable[str] \| None = None,
93	file_mappings: Mapping[str, str] \| None = None,
94	files: Iterable[str] \| None = None,
95	default_action: JarDuplicateAction \| None = None,
96	policies: Iterable[tuple[str, str \| JarDuplicateAction]] \| None = None,
97	skip: Iterable[str] \| None = None,
98	compress: bool = False,
99	update: bool = False,
100	) -> None:
UNCOV 101	_file_mappings = {(file_mappings or {}), ({f: f for f in (files or [])})}	×
102
UNCOV 103	object.__setattr__(self, "jar_name", jar_name)	×
UNCOV 104	object.__setattr__(self, "digest", digest)	×
UNCOV 105	object.__setattr__(self, "main_class", main_class)	×
UNCOV 106	object.__setattr__(self, "manifest", manifest)	×
UNCOV 107	object.__setattr__(self, "classpath_entries", tuple(classpath_entries or ()))	×
UNCOV 108	object.__setattr__(self, "jars", tuple(jars or ()))	×
UNCOV 109	object.__setattr__(self, "file_mappings", FrozenDict(_file_mappings))	×
UNCOV 110	object.__setattr__(self, "default_action", default_action)	×
UNCOV 111	object.__setattr__(self, "policies", tuple(JarToolRequest.__parse_policies(policies or ())))	×
UNCOV 112	object.__setattr__(self, "skip", tuple(skip or ()))	×
UNCOV 113	object.__setattr__(self, "compress", compress)	×
UNCOV 114	object.__setattr__(self, "update", update)	×
115
116	@staticmethod	4✔
117	def __parse_policies(	4✔
118	policies: Iterable[tuple[str, str \| JarDuplicateAction]],
119	) -> Iterable[tuple[str, JarDuplicateAction]]:
UNCOV 120	return [	×
121	(
122	pattern,
123	(
124	action
125	if isinstance(action, JarDuplicateAction)
126	else JarDuplicateAction(action.lower())
127	),
128	)
129	for (pattern, action) in policies
130	]
131
132
133	_JAR_TOOL_MAIN_CLASS = "org.pantsbuild.tools.jar.Main"	4✔
134
135
136	@dataclass(frozen=True)	4✔
137	class JarToolCompiledClassfiles:	4✔
138	digest: Digest	4✔
139
140
141	@rule	4✔
142	async def run_jar_tool(	4✔
143	request: JarToolRequest, jdk: InternalJdk, tool: JarTool, jar_tool: JarToolCompiledClassfiles
144	) -> Digest:
145	output_prefix = "__out"	×
146	output_jarname = os.path.join(output_prefix, request.jar_name)	×
147
148	tool_classpath, empty_output_digest = await concurrently(	×
149	materialize_classpath_for_tool(
150	ToolClasspathRequest(lockfile=GenerateJvmLockfileFromTool.create(tool))
151	),
152	create_digest(CreateDigest([Directory(output_prefix)])),
153	)
154
155	toolcp_prefix = "__toolcp"	×
156	jartoolcp_prefix = "__jartoolcp"	×
157	input_prefix = "__in"	×
158	immutable_input_digests = {	×
159	toolcp_prefix: tool_classpath.digest,
160	jartoolcp_prefix: jar_tool.digest,
161	input_prefix: request.digest,
162	}
163
164	policies = ",".join(	×
165	f"{pattern}={action.value.upper()}" for (pattern, action) in request.policies
166	)
167	file_mappings = ",".join(	×
168	f"{os.path.join(input_prefix, fs_path)}={jar_path}"
169	for fs_path, jar_path in request.file_mappings.items()
170	)
171
172	tool_process = JvmProcess(	×
173	jdk=jdk,
174	argv=[
175	_JAR_TOOL_MAIN_CLASS,
176	output_jarname,
177	*((f"-main={request.main_class}",) if request.main_class else ()),
178	*(
179	(f"-classpath={','.join(request.classpath_entries)}",)
180	if request.classpath_entries
181	else ()
182	),
183	*(
184	(f"-manifest={os.path.join(input_prefix, request.manifest)}",)
185	if request.manifest
186	else ()
187	),
188	*(
189	(f"-jars={','.join([os.path.join(input_prefix, jar) for jar in request.jars])}",)
190	if request.jars
191	else ()
192	),
193	*((f"-files={file_mappings}",) if file_mappings else ()),
194	*(
195	(f"-default_action={request.default_action.value.upper()}",)
196	if request.default_action
197	else ()
198	),
199	*((f"-policies={policies}",) if policies else ()),
200	*((f"-skip={','.join(request.skip)}",) if request.skip else ()),
201	*(("-compress",) if request.compress else ()),
202	*(("-update",) if request.update else ()),
203	],
204	classpath_entries=[*tool_classpath.classpath_entries(toolcp_prefix), jartoolcp_prefix],
205	input_digest=empty_output_digest,
206	extra_immutable_input_digests=immutable_input_digests,
207	extra_nailgun_keys=immutable_input_digests.keys(),
208	description=f"Building jar {request.jar_name}",
209	output_directories=(output_prefix,),
210	level=LogLevel.DEBUG,
211	)
212
213	result = await execute_process_or_raise(**implicitly({tool_process: JvmProcess}))	×
214	return await remove_prefix(RemovePrefix(result.output_digest, output_prefix))	×
215
216
217	_JAR_TOOL_SRC_PACKAGES = ["args4j", "jar_tool_source"]	4✔
218
219
220	def _load_jar_tool_sources() -> list[FileContent]:	4✔
221	parent_module = ".".join(__name__.split(".")[:-1])	×
222	result = []	×
223	for package in _JAR_TOOL_SRC_PACKAGES:	×
224	# pkg_path = package.replace(".", os.path.sep)
225	# relative_folder = os.path.join("src", pkg_path)
226	for resource in importlib.resources.files(parent_module).joinpath(package).iterdir():	×
227	if not resource.is_file():	×
228	continue	×
229	result.append(	×
230	FileContent(
231	path=os.path.join(package, resource.name),
232	content=resource.read_bytes(),
233	)
234	)
235	return result	×
236
237
238	# TODO(13879): Consolidate compilation of wrapper binaries to common rules.
239	@rule	4✔
240	async def build_jar_tool(jdk: InternalJdk, tool: JarTool) -> JarToolCompiledClassfiles:	4✔
241	source_digest = await create_digest(CreateDigest(_load_jar_tool_sources()))	×
242
243	dest_dir = "classfiles"	×
244	materialized_classpath, java_subset_digest, empty_dest_dir = await concurrently(	×
245	materialize_classpath_for_tool(
246	ToolClasspathRequest(
247	prefix="__toolcp", lockfile=GenerateJvmLockfileFromTool.create(tool)
248	)
249	),
250	digest_subset_to_digest(
251	DigestSubset(
252	source_digest,
253	PathGlobs(
254	["*/.java"],
255	glob_match_error_behavior=GlobMatchErrorBehavior.error,
256	description_of_origin="jar tool sources",
257	),
258	)
259	),
260	create_digest(CreateDigest([Directory(path=dest_dir)])),
261	)
262
263	merged_digest, src_entries = await concurrently(	×
264	merge_digests(MergeDigests([materialized_classpath.digest, source_digest, empty_dest_dir])),
265	get_digest_entries(java_subset_digest),
266	)
267
268	compile_result = await execute_process_or_raise(	×
269	**implicitly(
270	JvmProcess(
271	jdk=jdk,
272	classpath_entries=[f"{jdk.java_home}/lib/tools.jar"],
273	argv=[
274	"com.sun.tools.javac.Main",
275	"-cp",
276	":".join(materialized_classpath.classpath_entries()),
277	"-d",
278	dest_dir,
279	*[entry.path for entry in src_entries if isinstance(entry, FileEntry)],
280	],
281	input_digest=merged_digest,
282	output_directories=(dest_dir,),
283	description="Compile jar-tool sources using javac.",
284	level=LogLevel.DEBUG,
285	use_nailgun=False,
286	)
287	)
288	)
289
290	stripped_classfiles_digest = await remove_prefix(	×
291	RemovePrefix(compile_result.output_digest, dest_dir)
292	)
293	return JarToolCompiledClassfiles(digest=stripped_classfiles_digest)	×
294
295
296	def rules():	4✔
297	return [	4✔
298	*collect_rules(),
299	*coursier_fetch_rules(),
300	*jdk_rules(),
301	*jvm_tool_rules(),
302	UnionRule(ExportableTool, JarTool),
303	]

pantsbuild / pants / 20147226056

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous