18791134616

Committed 24 Oct 2025 08:18PM UTC coverage: 75.519% (-4.8%) from 80.282%

Build # 18791134616

Build Type

Pull #22794

github

Committed by

web-flow

Commit Message

Merge 098c595a0 into 7971a20bf

Pull Request Pull Request #22794: Use self-hosted MacOS Intel runner

Run Details

65803 of 87134 relevant lines covered (75.52%)

3.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.74

/src/python/pants/jvm/jdk_rules.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import dataclasses
import logging
import os
import re
import shlex
import textwrap
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from enum import Enum
from typing import ClassVar

from pants.core.environments.target_types import EnvironmentTarget
from pants.core.util_rules.system_binaries import BashBinary, LnBinary
from pants.engine.fs import CreateDigest, Digest, FileContent, FileDigest, MergeDigests
from pants.engine.intrinsics import create_digest, execute_process, merge_digests
from pants.engine.process import Process, ProcessCacheScope
from pants.engine.rules import collect_rules, implicitly, rule
from pants.engine.target import CoarsenedTarget
from pants.jvm.compile import ClasspathEntry
from pants.jvm.resolve.coordinate import Coordinate, Coordinates
from pants.jvm.resolve.coursier_fetch import CoursierLockfileEntry, coursier_fetch_one_coord
from pants.jvm.resolve.coursier_setup import Coursier
from pants.jvm.subsystems import JvmSubsystem
from pants.jvm.target_types import JvmJdkField
from pants.option.global_options import GlobalOptions
from pants.util.docutil import bin_name
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel
from pants.util.meta import classproperty
from pants.util.strutil import fmt_memory_size, softwrap

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class Nailgun:
    classpath_entry: ClasspathEntry


class DefaultJdk(Enum):
    SYSTEM = "system"
    SOURCE_DEFAULT = "source_default"


@dataclass(frozen=True)
class JdkRequest:
    """Request for a JDK with a specific major version, or a default (`--jvm-jdk` or System)."""

    version: str | DefaultJdk

    @classproperty
    def SYSTEM(cls) -> JdkRequest:
        return JdkRequest(DefaultJdk.SYSTEM)

    @classproperty
    def SOURCE_DEFAULT(cls) -> JdkRequest:
        return JdkRequest(DefaultJdk.SOURCE_DEFAULT)

    @staticmethod
    def from_field(field: JvmJdkField) -> JdkRequest:
        version = field.value
        if version == "system":
            return JdkRequest.SYSTEM
        return JdkRequest(version) if version is not None else JdkRequest.SOURCE_DEFAULT

    @staticmethod
    def from_target(target: CoarsenedTarget) -> JdkRequest:
        fields = [t[JvmJdkField] for t in target.members if t.has_field(JvmJdkField)]

        if not fields:
            raise ValueError(
                f"Cannot construct a JDK request for {target}, since none of its "
                f"members have a `{JvmJdkField.alias}=` field:\n{target.bullet_list()}"
            )

        field = fields[0]
        if not all(f.value == field.value for f in fields):
            values = {f.value for f in fields}
            raise ValueError(
                f"The members of {target} had mismatched values of the "
                f"`{JvmJdkField.alias}=` field ({values}):\n{target.bullet_list()}"
            )

        return JdkRequest.from_field(field)


@dataclass(frozen=True)
class JdkEnvironment:
    _digest: Digest
    nailgun_jar: str
    coursier: Coursier
    jre_major_version: int
    global_jvm_options: tuple[str, ...]
    java_home_command: str

    bin_dir: ClassVar[str] = "__jdk"
    jdk_preparation_script: ClassVar[str] = f"{bin_dir}/jdk.sh"
    java_home: ClassVar[str] = "__java_home"

    def args(
        self, bash: BashBinary, classpath_entries: Iterable[str], chroot: str | None = None
    ) -> tuple[str, ...]:
        def in_chroot(path: str) -> str:
            if not chroot:
                return path
            return os.path.join(chroot, path)

        return (
            bash.path,
            in_chroot(self.jdk_preparation_script),
            f"{self.java_home}/bin/java",
            "-cp",
            ":".join([in_chroot(self.nailgun_jar), *classpath_entries]),
        )

    @property
    def env(self) -> dict[str, str]:
        return self.coursier.env

    @property
    def append_only_caches(self) -> dict[str, str]:
        return self.coursier.append_only_caches

    @property
    def immutable_input_digests(self) -> dict[str, Digest]:
        return {**self.coursier.immutable_input_digests, self.bin_dir: self._digest}


@dataclass(frozen=True)
class InternalJdk(JdkEnvironment):
    """The JDK configured for internal Pants usage, rather than for matching source compatibility.

    The InternalJdk should only be used in situations where no classfiles are required for a user's
    firstparty or thirdparty code (such as for codegen, or analysis of source files).
    """

    @classmethod
    def from_jdk_environment(cls, env: JdkEnvironment) -> InternalJdk:
        return cls(
            env._digest,
            env.nailgun_jar,
            env.coursier,
            env.jre_major_version,
            env.global_jvm_options,
            env.java_home_command,
        )


VERSION_REGEX = re.compile(r"version \"(.+?)\"")


def parse_jre_major_version(version_lines: str) -> int | None:
    for line in version_lines.splitlines():
        m = VERSION_REGEX.search(line)
        if m:
            major_version, _, _ = m[1].partition(".")
            return int(major_version)
    return None


@rule
async def fetch_nailgun() -> Nailgun:
    nailgun = await coursier_fetch_one_coord(
        CoursierLockfileEntry(
            coord=Coordinate.from_coord_str("com.martiansoftware:nailgun-server:0.9.1"),
            file_name="com.martiansoftware_nailgun-server_0.9.1.jar",
            direct_dependencies=Coordinates(),
            dependencies=Coordinates(),
            file_digest=FileDigest(
                fingerprint="4518faa6bf4bd26fccdc4d85e1625dc679381a08d56872d8ad12151dda9cef25",
                serialized_bytes_length=32927,
            ),
        )
    )

    return Nailgun(nailgun)


@rule
async def prepare_jdk_environment(
    jvm: JvmSubsystem,
    jvm_env_aware: JvmSubsystem.EnvironmentAware,
    coursier: Coursier,
    nailgun_: Nailgun,
    bash: BashBinary,
    ln: LnBinary,
    request: JdkRequest,
    env_target: EnvironmentTarget,
) -> JdkEnvironment:
    nailgun = nailgun_.classpath_entry

    version = request.version
    if version == DefaultJdk.SOURCE_DEFAULT:
        version = jvm.jdk

    # TODO: add support for system JDKs with specific version
    if version is DefaultJdk.SYSTEM:
        coursier_jdk_option = "--system-jvm"
    else:
        coursier_jdk_option = f"--jvm={version}"

    if not coursier.jvm_index:
        coursier_options = ["java-home", coursier_jdk_option]
    else:
        jvm_index_option = f"--jvm-index={coursier.jvm_index}"
        coursier_options = ["java-home", jvm_index_option, coursier_jdk_option]

    # TODO(#16104) This argument re-writing code should use the native {chroot} support.
    # See also `run` for other argument re-writing code.
    def prefixed(arg: str) -> str:
        quoted = shlex.quote(arg)
        if arg.startswith("__"):
            return f"${{PANTS_INTERNAL_ABSOLUTE_PREFIX}}{quoted}"
        else:
            return quoted

    optionally_prefixed_coursier_args = [prefixed(arg) for arg in coursier.args(coursier_options)]
    # NB: We `set +e` in the subshell to ensure that it exits as well.
    #  see https://unix.stackexchange.com/a/23099
    java_home_command = " ".join(("set +e;", *optionally_prefixed_coursier_args))

    env = {
        "PANTS_INTERNAL_ABSOLUTE_PREFIX": "",
        **coursier.env,
    }

    java_version_result = await execute_process(
        Process(
            argv=(
                bash.path,
                "-c",
                f"$({java_home_command})/bin/java -version",
            ),
            append_only_caches=coursier.append_only_caches,
            immutable_input_digests=coursier.immutable_input_digests,
            env=env,
            description=f"Ensure download of JDK {coursier_jdk_option}.",
            cache_scope=env_target.executable_search_path_cache_scope(),
            level=LogLevel.DEBUG,
        ),
        **implicitly(),
    )

    if java_version_result.exit_code != 0:
        raise ValueError(
            f"Failed to locate Java for JDK `{version}`:\n"
            f"{java_version_result.stderr.decode('utf-8')}"
        )

    java_version = java_version_result.stderr.decode("utf-8").strip()
    jre_major_version = parse_jre_major_version(java_version)
    if not jre_major_version:
        raise ValueError(
            "Pants was unable to parse the output of `java -version` for JDK "
            f"`{request.version}`. Please open an issue at "
            "https://github.com/pantsbuild/pants/issues/new/choose with the following output:\n\n"
            f"{java_version}"
        )

    # TODO: Locate `ln`.
    version_comment = "\n".join(f"# {line}" for line in java_version.splitlines())
    ln_path = shlex.quote(ln.path)
    jdk_preparation_script = textwrap.dedent(  # noqa: PNT20
        f"""\
        # pants javac script using Coursier {coursier_jdk_option}. `java -version`:"
        {version_comment}
        set -eu

        {ln_path} -s "$({java_home_command})" "${{PANTS_INTERNAL_ABSOLUTE_PREFIX}}{JdkEnvironment.java_home}"
        exec "$@"
        """
    )
    jdk_preparation_script_digest = await create_digest(
        CreateDigest(
            [
                FileContent(
                    os.path.basename(JdkEnvironment.jdk_preparation_script),
                    jdk_preparation_script.encode("utf-8"),
                    is_executable=True,
                ),
            ]
        ),
    )

    return JdkEnvironment(
        _digest=await merge_digests(
            MergeDigests(
                [
                    jdk_preparation_script_digest,
                    nailgun.digest,
                ]
            ),
        ),
        global_jvm_options=jvm_env_aware.global_options,
        nailgun_jar=os.path.join(JdkEnvironment.bin_dir, nailgun.filenames[0]),
        coursier=coursier,
        jre_major_version=jre_major_version,
        java_home_command=java_home_command,
    )


@rule
async def internal_jdk(jvm: JvmSubsystem) -> InternalJdk:
    """Creates a `JdkEnvironment` object based on the JVM subsystem options.

    This is used for providing a predictable JDK version for Pants' internal usage rather than for
    matching compatibility with source files (e.g. compilation/testing).
    """

    request = JdkRequest(jvm.tool_jdk) if jvm.tool_jdk is not None else JdkRequest.SYSTEM
    env = await prepare_jdk_environment(**implicitly({request: JdkRequest}))
    return InternalJdk.from_jdk_environment(env)


@dataclass(frozen=True)
class JvmProcess:
    jdk: JdkEnvironment
    argv: tuple[str, ...]
    classpath_entries: tuple[str, ...]
    input_digest: Digest
    description: str = dataclasses.field(compare=False)
    level: LogLevel
    extra_jvm_options: tuple[str, ...]
    extra_nailgun_keys: tuple[str, ...]
    output_files: tuple[str, ...]
    output_directories: tuple[str, ...]
    timeout_seconds: int | float | None
    extra_immutable_input_digests: FrozenDict[str, Digest]
    extra_env: FrozenDict[str, str]
    cache_scope: ProcessCacheScope | None
    use_nailgun: bool
    remote_cache_speculation_delay: int | None

    def __init__(
        self,
        jdk: JdkEnvironment,
        argv: Iterable[str],
        classpath_entries: Iterable[str],
        input_digest: Digest,
        description: str,
        level: LogLevel = LogLevel.INFO,
        extra_jvm_options: Iterable[str] | None = None,
        extra_nailgun_keys: Iterable[str] | None = None,
        output_files: Iterable[str] | None = None,
        output_directories: Iterable[str] | None = None,
        extra_immutable_input_digests: Mapping[str, Digest] | None = None,
        extra_env: Mapping[str, str] | None = None,
        timeout_seconds: int | float | None = None,
        cache_scope: ProcessCacheScope | None = None,
        use_nailgun: bool = True,
        remote_cache_speculation_delay: int | None = None,
    ):
        object.__setattr__(self, "jdk", jdk)
        object.__setattr__(self, "argv", tuple(argv))
        object.__setattr__(self, "classpath_entries", tuple(classpath_entries))
        object.__setattr__(self, "input_digest", input_digest)
        object.__setattr__(self, "description", description)
        object.__setattr__(self, "level", level)
        object.__setattr__(self, "extra_jvm_options", tuple(extra_jvm_options or ()))
        object.__setattr__(self, "extra_nailgun_keys", tuple(extra_nailgun_keys or ()))
        object.__setattr__(self, "output_files", tuple(output_files or ()))
        object.__setattr__(self, "output_directories", tuple(output_directories or ()))
        object.__setattr__(self, "timeout_seconds", timeout_seconds)
        object.__setattr__(self, "cache_scope", cache_scope)
        object.__setattr__(
            self, "extra_immutable_input_digests", FrozenDict(extra_immutable_input_digests or {})
        )
        object.__setattr__(self, "extra_env", FrozenDict(extra_env or {}))
        object.__setattr__(self, "use_nailgun", use_nailgun)
        object.__setattr__(self, "remote_cache_speculation_delay", remote_cache_speculation_delay)

        self.__post_init__()

    def __post_init__(self):
        if not self.use_nailgun and self.extra_nailgun_keys:
            raise AssertionError(
                "`JvmProcess` specified nailgun keys, but has `use_nailgun=False`. Either "
                "specify `extra_nailgun_keys=None` or `use_nailgun=True`."
            )


_JVM_HEAP_SIZE_UNITS = ["", "k", "m", "g"]


@rule
async def jvm_process(
    bash: BashBinary, request: JvmProcess, jvm: JvmSubsystem, global_options: GlobalOptions
) -> Process:
    jdk = request.jdk

    immutable_input_digests = {
        **jdk.immutable_input_digests,
        **request.extra_immutable_input_digests,
    }
    env = {
        "PANTS_INTERNAL_ABSOLUTE_PREFIX": "",
        **jdk.env,
        **request.extra_env,
    }

    def valid_jvm_opt(opt: str) -> str:
        if opt.startswith("-Xmx"):
            raise ValueError(
                softwrap(
                    f"""
                    Invalid value for JVM options: {opt}.

                    For setting a maximum heap size for the JVM child processes, use
                    `[GLOBAL].process_per_child_memory_usage` option instead.

                    Run `{bin_name()} help-advanced global` for more information.
                    """
                )
            )
        return opt

    max_heap_size = fmt_memory_size(
        global_options.process_per_child_memory_usage, units=_JVM_HEAP_SIZE_UNITS
    )
    jvm_user_options = [*jdk.global_jvm_options, *request.extra_jvm_options]
    jvm_options = [
        f"-Xmx{max_heap_size}",
        *[valid_jvm_opt(opt) for opt in jvm_user_options],
    ]

    use_nailgun = []
    if request.use_nailgun:
        use_nailgun = [*jdk.immutable_input_digests, *request.extra_nailgun_keys]

    remote_cache_speculation_delay_millis = 0
    if request.remote_cache_speculation_delay is not None:
        remote_cache_speculation_delay_millis = request.remote_cache_speculation_delay
    elif request.use_nailgun:
        remote_cache_speculation_delay_millis = jvm.nailgun_remote_cache_speculation_delay

    return Process(
        [*jdk.args(bash, request.classpath_entries), *jvm_options, *request.argv],
        input_digest=request.input_digest,
        immutable_input_digests=immutable_input_digests,
        use_nailgun=use_nailgun,
        description=request.description,
        level=request.level,
        output_directories=request.output_directories,
        env=env,
        timeout_seconds=request.timeout_seconds,
        append_only_caches=jdk.append_only_caches,
        output_files=request.output_files,
        cache_scope=request.cache_scope or ProcessCacheScope.SUCCESSFUL,
        remote_cache_speculation_delay_millis=remote_cache_speculation_delay_millis,
    )


def rules():
    return collect_rules()

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	10✔
5
6	import dataclasses	10✔
7	import logging	10✔
8	import os	10✔
9	import re	10✔
10	import shlex	10✔
11	import textwrap	10✔
12	from collections.abc import Iterable, Mapping	10✔
13	from dataclasses import dataclass	10✔
14	from enum import Enum	10✔
15	from typing import ClassVar	10✔
16
17	from pants.core.environments.target_types import EnvironmentTarget	10✔
18	from pants.core.util_rules.system_binaries import BashBinary, LnBinary	10✔
19	from pants.engine.fs import CreateDigest, Digest, FileContent, FileDigest, MergeDigests	10✔
20	from pants.engine.intrinsics import create_digest, execute_process, merge_digests	10✔
21	from pants.engine.process import Process, ProcessCacheScope	10✔
22	from pants.engine.rules import collect_rules, implicitly, rule	10✔
23	from pants.engine.target import CoarsenedTarget	10✔
24	from pants.jvm.compile import ClasspathEntry	10✔
25	from pants.jvm.resolve.coordinate import Coordinate, Coordinates	10✔
26	from pants.jvm.resolve.coursier_fetch import CoursierLockfileEntry, coursier_fetch_one_coord	10✔
27	from pants.jvm.resolve.coursier_setup import Coursier	10✔
28	from pants.jvm.subsystems import JvmSubsystem	10✔
29	from pants.jvm.target_types import JvmJdkField	10✔
30	from pants.option.global_options import GlobalOptions	10✔
31	from pants.util.docutil import bin_name	10✔
32	from pants.util.frozendict import FrozenDict	10✔
33	from pants.util.logging import LogLevel	10✔
34	from pants.util.meta import classproperty	10✔
35	from pants.util.strutil import fmt_memory_size, softwrap	10✔
36
37	logger = logging.getLogger(__name__)	10✔
38
39
40	@dataclass(frozen=True)	10✔
41	class Nailgun:	10✔
42	classpath_entry: ClasspathEntry	10✔
43
44
45	class DefaultJdk(Enum):	10✔
46	SYSTEM = "system"	10✔
47	SOURCE_DEFAULT = "source_default"	10✔
48
49
50	@dataclass(frozen=True)	10✔
51	class JdkRequest:	10✔
52	"""Request for a JDK with a specific major version, or a default (`--jvm-jdk` or System)."""
53
54	version: str \| DefaultJdk	10✔
55
56	@classproperty	10✔
57	def SYSTEM(cls) -> JdkRequest:	10✔
58	return JdkRequest(DefaultJdk.SYSTEM)	×
59
60	@classproperty	10✔
61	def SOURCE_DEFAULT(cls) -> JdkRequest:	10✔
62	return JdkRequest(DefaultJdk.SOURCE_DEFAULT)	×
63
64	@staticmethod	10✔
65	def from_field(field: JvmJdkField) -> JdkRequest:	10✔
66	version = field.value	×
67	if version == "system":	×
68	return JdkRequest.SYSTEM	×
69	return JdkRequest(version) if version is not None else JdkRequest.SOURCE_DEFAULT	×
70
71	@staticmethod	10✔
72	def from_target(target: CoarsenedTarget) -> JdkRequest:	10✔
73	fields = [t[JvmJdkField] for t in target.members if t.has_field(JvmJdkField)]	×
74
75	if not fields:	×
76	raise ValueError(	×
77	f"Cannot construct a JDK request for {target}, since none of its "
78	f"members have a `{JvmJdkField.alias}=` field:\n{target.bullet_list()}"
79	)
80
81	field = fields[0]	×
82	if not all(f.value == field.value for f in fields):	×
83	values = {f.value for f in fields}	×
84	raise ValueError(	×
85	f"The members of {target} had mismatched values of the "
86	f"`{JvmJdkField.alias}=` field ({values}):\n{target.bullet_list()}"
87	)
88
89	return JdkRequest.from_field(field)	×
90
91
92	@dataclass(frozen=True)	10✔
93	class JdkEnvironment:	10✔
94	_digest: Digest	10✔
95	nailgun_jar: str	10✔
96	coursier: Coursier	10✔
97	jre_major_version: int	10✔
98	global_jvm_options: tuple[str, ...]	10✔
99	java_home_command: str	10✔
100
101	bin_dir: ClassVar[str] = "__jdk"	10✔
102	jdk_preparation_script: ClassVar[str] = f"{bin_dir}/jdk.sh"	10✔
103	java_home: ClassVar[str] = "__java_home"	10✔
104
105	def args(	10✔
106	self, bash: BashBinary, classpath_entries: Iterable[str], chroot: str \| None = None
107	) -> tuple[str, ...]:
108	def in_chroot(path: str) -> str:	×
109	if not chroot:	×
110	return path	×
111	return os.path.join(chroot, path)	×
112
113	return (	×
114	bash.path,
115	in_chroot(self.jdk_preparation_script),
116	f"{self.java_home}/bin/java",
117	"-cp",
118	":".join([in_chroot(self.nailgun_jar), *classpath_entries]),
119	)
120
121	@property	10✔
122	def env(self) -> dict[str, str]:	10✔
123	return self.coursier.env	×
124
125	@property	10✔
126	def append_only_caches(self) -> dict[str, str]:	10✔
127	return self.coursier.append_only_caches	×
128
129	@property	10✔
130	def immutable_input_digests(self) -> dict[str, Digest]:	10✔
131	return {**self.coursier.immutable_input_digests, self.bin_dir: self._digest}	×
132
133
134	@dataclass(frozen=True)	10✔
135	class InternalJdk(JdkEnvironment):	10✔
136	"""The JDK configured for internal Pants usage, rather than for matching source compatibility.
137
138	The InternalJdk should only be used in situations where no classfiles are required for a user's
139	firstparty or thirdparty code (such as for codegen, or analysis of source files).
140	"""
141
142	@classmethod	10✔
143	def from_jdk_environment(cls, env: JdkEnvironment) -> InternalJdk:	10✔
144	return cls(	×
145	env._digest,
146	env.nailgun_jar,
147	env.coursier,
148	env.jre_major_version,
149	env.global_jvm_options,
150	env.java_home_command,
151	)
152
153
154	VERSION_REGEX = re.compile(r"version \"(.+?)\"")	10✔
155
156
157	def parse_jre_major_version(version_lines: str) -> int \| None:	10✔
158	for line in version_lines.splitlines():	×
159	m = VERSION_REGEX.search(line)	×
160	if m:	×
161	major_version, _, _ = m[1].partition(".")	×
162	return int(major_version)	×
163	return None	×
164
165
166	@rule	10✔
167	async def fetch_nailgun() -> Nailgun:	10✔
168	nailgun = await coursier_fetch_one_coord(	×
169	CoursierLockfileEntry(
170	coord=Coordinate.from_coord_str("com.martiansoftware:nailgun-server:0.9.1"),
171	file_name="com.martiansoftware_nailgun-server_0.9.1.jar",
172	direct_dependencies=Coordinates(),
173	dependencies=Coordinates(),
174	file_digest=FileDigest(
175	fingerprint="4518faa6bf4bd26fccdc4d85e1625dc679381a08d56872d8ad12151dda9cef25",
176	serialized_bytes_length=32927,
177	),
178	)
179	)
180
181	return Nailgun(nailgun)	×
182
183
184	@rule	10✔
185	async def prepare_jdk_environment(	10✔
186	jvm: JvmSubsystem,
187	jvm_env_aware: JvmSubsystem.EnvironmentAware,
188	coursier: Coursier,
189	nailgun_: Nailgun,
190	bash: BashBinary,
191	ln: LnBinary,
192	request: JdkRequest,
193	env_target: EnvironmentTarget,
194	) -> JdkEnvironment:
195	nailgun = nailgun_.classpath_entry	×
196
197	version = request.version	×
198	if version == DefaultJdk.SOURCE_DEFAULT:	×
199	version = jvm.jdk	×
200
201	# TODO: add support for system JDKs with specific version
202	if version is DefaultJdk.SYSTEM:	×
203	coursier_jdk_option = "--system-jvm"	×
204	else:
205	coursier_jdk_option = f"--jvm={version}"	×
206
207	if not coursier.jvm_index:	×
208	coursier_options = ["java-home", coursier_jdk_option]	×
209	else:
210	jvm_index_option = f"--jvm-index={coursier.jvm_index}"	×
211	coursier_options = ["java-home", jvm_index_option, coursier_jdk_option]	×
212
213	# TODO(#16104) This argument re-writing code should use the native {chroot} support.
214	# See also `run` for other argument re-writing code.
215	def prefixed(arg: str) -> str:	×
216	quoted = shlex.quote(arg)	×
217	if arg.startswith("__"):	×
218	return f"${{PANTS_INTERNAL_ABSOLUTE_PREFIX}}{quoted}"	×
219	else:
220	return quoted	×
221
222	optionally_prefixed_coursier_args = [prefixed(arg) for arg in coursier.args(coursier_options)]	×
223	# NB: We `set +e` in the subshell to ensure that it exits as well.
224	# see https://unix.stackexchange.com/a/23099
225	java_home_command = " ".join(("set +e;", *optionally_prefixed_coursier_args))	×
226
227	env = {	×
228	"PANTS_INTERNAL_ABSOLUTE_PREFIX": "",
229	**coursier.env,
230	}
231
232	java_version_result = await execute_process(	×
233	Process(
234	argv=(
235	bash.path,
236	"-c",
237	f"$({java_home_command})/bin/java -version",
238	),
239	append_only_caches=coursier.append_only_caches,
240	immutable_input_digests=coursier.immutable_input_digests,
241	env=env,
242	description=f"Ensure download of JDK {coursier_jdk_option}.",
243	cache_scope=env_target.executable_search_path_cache_scope(),
244	level=LogLevel.DEBUG,
245	),
246	**implicitly(),
247	)
248
249	if java_version_result.exit_code != 0:	×
250	raise ValueError(	×
251	f"Failed to locate Java for JDK `{version}`:\n"
252	f"{java_version_result.stderr.decode('utf-8')}"
253	)
254
255	java_version = java_version_result.stderr.decode("utf-8").strip()	×
256	jre_major_version = parse_jre_major_version(java_version)	×
257	if not jre_major_version:	×
258	raise ValueError(	×
259	"Pants was unable to parse the output of `java -version` for JDK "
260	f"`{request.version}`. Please open an issue at "
261	"https://github.com/pantsbuild/pants/issues/new/choose with the following output:\n\n"
262	f"{java_version}"
263	)
264
265	# TODO: Locate `ln`.
266	version_comment = "\n".join(f"# {line}" for line in java_version.splitlines())	×
267	ln_path = shlex.quote(ln.path)	×
268	jdk_preparation_script = textwrap.dedent( # noqa: PNT20	×
269	f"""\
270	# pants javac script using Coursier {coursier_jdk_option}. `java -version`:"
271	{version_comment}
272	set -eu
273
274	{ln_path} -s "$({java_home_command})" "${{PANTS_INTERNAL_ABSOLUTE_PREFIX}}{JdkEnvironment.java_home}"
275	exec "$@"
276	"""
277	)
278	jdk_preparation_script_digest = await create_digest(	×
279	CreateDigest(
280	[
281	FileContent(
282	os.path.basename(JdkEnvironment.jdk_preparation_script),
283	jdk_preparation_script.encode("utf-8"),
284	is_executable=True,
285	),
286	]
287	),
288	)
289
290	return JdkEnvironment(	×
291	_digest=await merge_digests(
292	MergeDigests(
293	[
294	jdk_preparation_script_digest,
295	nailgun.digest,
296	]
297	),
298	),
299	global_jvm_options=jvm_env_aware.global_options,
300	nailgun_jar=os.path.join(JdkEnvironment.bin_dir, nailgun.filenames[0]),
301	coursier=coursier,
302	jre_major_version=jre_major_version,
303	java_home_command=java_home_command,
304	)
305
306
307	@rule	10✔
308	async def internal_jdk(jvm: JvmSubsystem) -> InternalJdk:	10✔
309	"""Creates a `JdkEnvironment` object based on the JVM subsystem options.
310
311	This is used for providing a predictable JDK version for Pants' internal usage rather than for
312	matching compatibility with source files (e.g. compilation/testing).
313	"""
314
315	request = JdkRequest(jvm.tool_jdk) if jvm.tool_jdk is not None else JdkRequest.SYSTEM	×
316	env = await prepare_jdk_environment(**implicitly({request: JdkRequest}))	×
317	return InternalJdk.from_jdk_environment(env)	×
318
319
320	@dataclass(frozen=True)	10✔
321	class JvmProcess:	10✔
322	jdk: JdkEnvironment	10✔
323	argv: tuple[str, ...]	10✔
324	classpath_entries: tuple[str, ...]	10✔
325	input_digest: Digest	10✔
326	description: str = dataclasses.field(compare=False)	10✔
327	level: LogLevel	10✔
328	extra_jvm_options: tuple[str, ...]	10✔
329	extra_nailgun_keys: tuple[str, ...]	10✔
330	output_files: tuple[str, ...]	10✔
331	output_directories: tuple[str, ...]	10✔
332	timeout_seconds: int \| float \| None	10✔
333	extra_immutable_input_digests: FrozenDict[str, Digest]	10✔
334	extra_env: FrozenDict[str, str]	10✔
335	cache_scope: ProcessCacheScope \| None	10✔
336	use_nailgun: bool	10✔
337	remote_cache_speculation_delay: int \| None	10✔
338
339	def __init__(	10✔
340	self,
341	jdk: JdkEnvironment,
342	argv: Iterable[str],
343	classpath_entries: Iterable[str],
344	input_digest: Digest,
345	description: str,
346	level: LogLevel = LogLevel.INFO,
347	extra_jvm_options: Iterable[str] \| None = None,
348	extra_nailgun_keys: Iterable[str] \| None = None,
349	output_files: Iterable[str] \| None = None,
350	output_directories: Iterable[str] \| None = None,
351	extra_immutable_input_digests: Mapping[str, Digest] \| None = None,
352	extra_env: Mapping[str, str] \| None = None,
353	timeout_seconds: int \| float \| None = None,
354	cache_scope: ProcessCacheScope \| None = None,
355	use_nailgun: bool = True,
356	remote_cache_speculation_delay: int \| None = None,
357	):
358	object.__setattr__(self, "jdk", jdk)	1✔
359	object.__setattr__(self, "argv", tuple(argv))	1✔
360	object.__setattr__(self, "classpath_entries", tuple(classpath_entries))	1✔
361	object.__setattr__(self, "input_digest", input_digest)	1✔
362	object.__setattr__(self, "description", description)	1✔
363	object.__setattr__(self, "level", level)	1✔
364	object.__setattr__(self, "extra_jvm_options", tuple(extra_jvm_options or ()))	1✔
365	object.__setattr__(self, "extra_nailgun_keys", tuple(extra_nailgun_keys or ()))	1✔
366	object.__setattr__(self, "output_files", tuple(output_files or ()))	1✔
367	object.__setattr__(self, "output_directories", tuple(output_directories or ()))	1✔
368	object.__setattr__(self, "timeout_seconds", timeout_seconds)	1✔
369	object.__setattr__(self, "cache_scope", cache_scope)	1✔
370	object.__setattr__(	1✔
371	self, "extra_immutable_input_digests", FrozenDict(extra_immutable_input_digests or {})
372	)
373	object.__setattr__(self, "extra_env", FrozenDict(extra_env or {}))	1✔
374	object.__setattr__(self, "use_nailgun", use_nailgun)	1✔
375	object.__setattr__(self, "remote_cache_speculation_delay", remote_cache_speculation_delay)	1✔
376
377	self.__post_init__()	1✔
378
379	def __post_init__(self):	10✔
380	if not self.use_nailgun and self.extra_nailgun_keys:	1✔
381	raise AssertionError(	×
382	"`JvmProcess` specified nailgun keys, but has `use_nailgun=False`. Either "
383	"specify `extra_nailgun_keys=None` or `use_nailgun=True`."
384	)
385
386
387	_JVM_HEAP_SIZE_UNITS = ["", "k", "m", "g"]	10✔
388
389
390	@rule	10✔
391	async def jvm_process(	10✔
392	bash: BashBinary, request: JvmProcess, jvm: JvmSubsystem, global_options: GlobalOptions
393	) -> Process:
394	jdk = request.jdk	×
395
396	immutable_input_digests = {	×
397	**jdk.immutable_input_digests,
398	**request.extra_immutable_input_digests,
399	}
400	env = {	×
401	"PANTS_INTERNAL_ABSOLUTE_PREFIX": "",
402	**jdk.env,
403	**request.extra_env,
404	}
405
406	def valid_jvm_opt(opt: str) -> str:	×
407	if opt.startswith("-Xmx"):	×
408	raise ValueError(	×
409	softwrap(
410	f"""
411	Invalid value for JVM options: {opt}.
412
413	For setting a maximum heap size for the JVM child processes, use
414	`[GLOBAL].process_per_child_memory_usage` option instead.
415
416	Run `{bin_name()} help-advanced global` for more information.
417	"""
418	)
419	)
420	return opt	×
421
422	max_heap_size = fmt_memory_size(	×
423	global_options.process_per_child_memory_usage, units=_JVM_HEAP_SIZE_UNITS
424	)
425	jvm_user_options = [jdk.global_jvm_options, request.extra_jvm_options]	×
426	jvm_options = [	×
427	f"-Xmx{max_heap_size}",
428	*[valid_jvm_opt(opt) for opt in jvm_user_options],
429	]
430
431	use_nailgun = []	×
432	if request.use_nailgun:	×
433	use_nailgun = [jdk.immutable_input_digests, request.extra_nailgun_keys]	×
434
435	remote_cache_speculation_delay_millis = 0	×
436	if request.remote_cache_speculation_delay is not None:	×
437	remote_cache_speculation_delay_millis = request.remote_cache_speculation_delay	×
438	elif request.use_nailgun:	×
439	remote_cache_speculation_delay_millis = jvm.nailgun_remote_cache_speculation_delay	×
440
441	return Process(	×
442	[jdk.args(bash, request.classpath_entries), jvm_options, *request.argv],
443	input_digest=request.input_digest,
444	immutable_input_digests=immutable_input_digests,
445	use_nailgun=use_nailgun,
446	description=request.description,
447	level=request.level,
448	output_directories=request.output_directories,
449	env=env,
450	timeout_seconds=request.timeout_seconds,
451	append_only_caches=jdk.append_only_caches,
452	output_files=request.output_files,
453	cache_scope=request.cache_scope or ProcessCacheScope.SUCCESSFUL,
454	remote_cache_speculation_delay_millis=remote_cache_speculation_delay_millis,
455	)
456
457
458	def rules():	10✔
459	return collect_rules()	10✔

pantsbuild / pants / 18791134616

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous