21365507481

Committed 26 Jan 2026 04:33PM UTC coverage: 80.263% (-0.006%) from 80.269%

Build # 21365507481

Build Type

Pull #23036

github

Committed by

web-flow

Commit Message

Merge 40511af22 into 09b8ecaa1

Pull Request Pull Request #23036: Fix non-deterministic JDK preparation script causing cache misses

Run Details

0 of 9 new or added lines in 1 file covered. (0.0%)

37 existing lines in 1 file now uncovered.

78764 of 98133 relevant lines covered (80.26%)

3.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.88

/src/python/pants/jvm/jdk_rules.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import dataclasses
import logging
import os
import re
import shlex
import textwrap
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from enum import Enum
from typing import ClassVar

from pants.core.environments.target_types import EnvironmentTarget
from pants.core.util_rules.system_binaries import BashBinary, LnBinary
from pants.engine.fs import CreateDigest, Digest, FileContent, FileDigest, MergeDigests
from pants.engine.intrinsics import create_digest, execute_process, merge_digests
from pants.engine.process import Process, ProcessCacheScope
from pants.engine.rules import collect_rules, implicitly, rule
from pants.engine.target import CoarsenedTarget
from pants.jvm.compile import ClasspathEntry
from pants.jvm.resolve.coordinate import Coordinate, Coordinates
from pants.jvm.resolve.coursier_fetch import CoursierLockfileEntry, coursier_fetch_one_coord
from pants.jvm.resolve.coursier_setup import Coursier
from pants.jvm.subsystems import JvmSubsystem
from pants.jvm.target_types import JvmJdkField
from pants.option.global_options import GlobalOptions
from pants.util.docutil import bin_name
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel
from pants.util.meta import classproperty
from pants.util.strutil import fmt_memory_size, softwrap

logger = logging.getLogger(__name__)


@dataclass(frozen=True)
class Nailgun:
    classpath_entry: ClasspathEntry


class DefaultJdk(Enum):
    SYSTEM = "system"
    SOURCE_DEFAULT = "source_default"


@dataclass(frozen=True)
class JdkRequest:
    """Request for a JDK with a specific major version, or a default (`--jvm-jdk` or System)."""

    version: str | DefaultJdk

    @classproperty
    def SYSTEM(cls) -> JdkRequest:
        return JdkRequest(DefaultJdk.SYSTEM)

    @classproperty
    def SOURCE_DEFAULT(cls) -> JdkRequest:
        return JdkRequest(DefaultJdk.SOURCE_DEFAULT)

    @staticmethod
    def from_field(field: JvmJdkField) -> JdkRequest:
        version = field.value
        if version == "system":
            return JdkRequest.SYSTEM
        return JdkRequest(version) if version is not None else JdkRequest.SOURCE_DEFAULT

    @staticmethod
    def from_target(target: CoarsenedTarget) -> JdkRequest:
        fields = [t[JvmJdkField] for t in target.members if t.has_field(JvmJdkField)]

        if not fields:
            raise ValueError(
                f"Cannot construct a JDK request for {target}, since none of its "
                f"members have a `{JvmJdkField.alias}=` field:\n{target.bullet_list()}"
            )

        field = fields[0]
        if not all(f.value == field.value for f in fields):
            values = {f.value for f in fields}
            raise ValueError(
                f"The members of {target} had mismatched values of the "
                f"`{JvmJdkField.alias}=` field ({values}):\n{target.bullet_list()}"
            )

        return JdkRequest.from_field(field)


@dataclass(frozen=True)
class JdkEnvironment:
    _digest: Digest
    nailgun_jar: str
    coursier: Coursier
    jre_major_version: int
    global_jvm_options: tuple[str, ...]
    java_home_command: str

    bin_dir: ClassVar[str] = "__jdk"
    jdk_preparation_script: ClassVar[str] = f"{bin_dir}/jdk.sh"
    java_home: ClassVar[str] = "__java_home"

    def args(
        self, bash: BashBinary, classpath_entries: Iterable[str], chroot: str | None = None
    ) -> tuple[str, ...]:
        def in_chroot(path: str) -> str:
            if not chroot:
                return path
            return os.path.join(chroot, path)

        return (
            bash.path,
            in_chroot(self.jdk_preparation_script),
            f"{self.java_home}/bin/java",
            "-cp",
            ":".join([in_chroot(self.nailgun_jar), *classpath_entries]),
        )

    @property
    def env(self) -> dict[str, str]:
        return self.coursier.env

    @property
    def append_only_caches(self) -> dict[str, str]:
        return self.coursier.append_only_caches

    @property
    def immutable_input_digests(self) -> dict[str, Digest]:
        return {**self.coursier.immutable_input_digests, self.bin_dir: self._digest}


@dataclass(frozen=True)
class InternalJdk(JdkEnvironment):
    """The JDK configured for internal Pants usage, rather than for matching source compatibility.

    The InternalJdk should only be used in situations where no classfiles are required for a user's
    firstparty or thirdparty code (such as for codegen, or analysis of source files).
    """

    @classmethod
    def from_jdk_environment(cls, env: JdkEnvironment) -> InternalJdk:
        return cls(
            env._digest,
            env.nailgun_jar,
            env.coursier,
            env.jre_major_version,
            env.global_jvm_options,
            env.java_home_command,
        )


VERSION_REGEX = re.compile(r"version \"(.+?)\"")


def parse_jre_major_version(version_lines: str) -> int | None:
    for line in version_lines.splitlines():
        m = VERSION_REGEX.search(line)
        if m:
            major_version, _, _ = m[1].partition(".")
            return int(major_version)
    return None


@rule
async def fetch_nailgun() -> Nailgun:
    nailgun = await coursier_fetch_one_coord(
        CoursierLockfileEntry(
            coord=Coordinate.from_coord_str("com.martiansoftware:nailgun-server:0.9.1"),
            file_name="com.martiansoftware_nailgun-server_0.9.1.jar",
            direct_dependencies=Coordinates(),
            dependencies=Coordinates(),
            file_digest=FileDigest(
                fingerprint="4518faa6bf4bd26fccdc4d85e1625dc679381a08d56872d8ad12151dda9cef25",
                serialized_bytes_length=32927,
            ),
        )
    )

    return Nailgun(nailgun)


@rule
async def prepare_jdk_environment(
    jvm: JvmSubsystem,
    jvm_env_aware: JvmSubsystem.EnvironmentAware,
    coursier: Coursier,
    nailgun_: Nailgun,
    bash: BashBinary,
    ln: LnBinary,
    request: JdkRequest,
    env_target: EnvironmentTarget,
) -> JdkEnvironment:
    nailgun = nailgun_.classpath_entry

    version = request.version
    if version == DefaultJdk.SOURCE_DEFAULT:
        version = jvm.jdk

    # TODO: add support for system JDKs with specific version
    if version is DefaultJdk.SYSTEM:
        coursier_jdk_option = "--system-jvm"
    else:
        coursier_jdk_option = f"--jvm={version}"

    if not coursier.jvm_index:
        coursier_options = ["java-home", coursier_jdk_option]
    else:
        jvm_index_option = f"--jvm-index={coursier.jvm_index}"
        coursier_options = ["java-home", jvm_index_option, coursier_jdk_option]

    # TODO(#16104) This argument re-writing code should use the native {chroot} support.
    # See also `run` for other argument re-writing code.
    def prefixed(arg: str) -> str:
        quoted = shlex.quote(arg)
        if arg.startswith("__"):
            return f"${{PANTS_INTERNAL_ABSOLUTE_PREFIX}}{quoted}"
        else:
            return quoted

    optionally_prefixed_coursier_args = [prefixed(arg) for arg in coursier.args(coursier_options)]
    # NB: We `set +e` in the subshell to ensure that it exits as well.
    #  see https://unix.stackexchange.com/a/23099
    java_home_command = " ".join(("set +e;", *optionally_prefixed_coursier_args))

    env = {
        "PANTS_INTERNAL_ABSOLUTE_PREFIX": "",
        **coursier.env,
    }

    java_version_result = await execute_process(
        Process(
            argv=(
                bash.path,
                "-c",
                f"$({java_home_command})/bin/java -version",
            ),
            append_only_caches=coursier.append_only_caches,
            immutable_input_digests=coursier.immutable_input_digests,
            env=env,
            description=f"Ensure download of JDK {coursier_jdk_option}.",
            cache_scope=env_target.executable_search_path_cache_scope(),
            level=LogLevel.DEBUG,
        ),
        **implicitly(),
    )

    if java_version_result.exit_code != 0:
        raise ValueError(
            f"Failed to locate Java for JDK `{version}`:\n"
            f"{java_version_result.stderr.decode('utf-8')}"
        )

    java_version_raw = java_version_result.stderr.decode("utf-8").strip()
    
    # Filter out Coursier download progress messages which are non-deterministic
    # and would cause cache key instability. Keep only actual java -version output.
    java_version_lines = []
    for line in java_version_raw.splitlines():
        line_lower = line.lower().strip()
        # Skip Coursier download progress messages
        if any(skip in line_lower for skip in [
            "downloading",
            "still downloading",
            "downloaded",
            "https://",
            "http://",
            "found ",
        ]):
            continue
        # Keep lines that look like java -version output
        if line.strip():
            java_version_lines.append(line)
    
    java_version = "\n".join(java_version_lines)
    jre_major_version = parse_jre_major_version(java_version)
    if not jre_major_version:
        raise ValueError(
            "Pants was unable to parse the output of `java -version` for JDK "
            f"`{request.version}`. Please open an issue at "
            "https://github.com/pantsbuild/pants/issues/new/choose with the following output:\n\n"
            f"{java_version}"
        )

    # TODO: Locate `ln`.
    version_comment = "\n".join(f"# {line}" for line in java_version.splitlines())
    ln_path = shlex.quote(ln.path)
    jdk_preparation_script = textwrap.dedent(  # noqa: PNT20
        f"""\
        # pants javac script using Coursier {coursier_jdk_option}. `java -version`:"
        {version_comment}
        set -eu

        {ln_path} -s "$({java_home_command})" "${{PANTS_INTERNAL_ABSOLUTE_PREFIX}}{JdkEnvironment.java_home}"
        exec "$@"
        """
    )
    jdk_preparation_script_digest = await create_digest(
        CreateDigest(
            [
                FileContent(
                    os.path.basename(JdkEnvironment.jdk_preparation_script),
                    jdk_preparation_script.encode("utf-8"),
                    is_executable=True,
                ),
            ]
        ),
    )

    return JdkEnvironment(
        _digest=await merge_digests(
            MergeDigests(
                [
                    jdk_preparation_script_digest,
                    nailgun.digest,
                ]
            ),
        ),
        global_jvm_options=jvm_env_aware.global_options,
        nailgun_jar=os.path.join(JdkEnvironment.bin_dir, nailgun.filenames[0]),
        coursier=coursier,
        jre_major_version=jre_major_version,
        java_home_command=java_home_command,
    )


@rule
async def internal_jdk(jvm: JvmSubsystem) -> InternalJdk:
    """Creates a `JdkEnvironment` object based on the JVM subsystem options.

    This is used for providing a predictable JDK version for Pants' internal usage rather than for
    matching compatibility with source files (e.g. compilation/testing).
    """

    request = JdkRequest(jvm.tool_jdk) if jvm.tool_jdk is not None else JdkRequest.SYSTEM
    env = await prepare_jdk_environment(**implicitly({request: JdkRequest}))
    return InternalJdk.from_jdk_environment(env)


@dataclass(frozen=True)
class JvmProcess:
    jdk: JdkEnvironment
    argv: tuple[str, ...]
    classpath_entries: tuple[str, ...]
    input_digest: Digest
    description: str = dataclasses.field(compare=False)
    level: LogLevel
    extra_jvm_options: tuple[str, ...]
    extra_nailgun_keys: tuple[str, ...]
    output_files: tuple[str, ...]
    output_directories: tuple[str, ...]
    timeout_seconds: int | float | None
    extra_immutable_input_digests: FrozenDict[str, Digest]
    extra_env: FrozenDict[str, str]
    cache_scope: ProcessCacheScope | None
    use_nailgun: bool
    remote_cache_speculation_delay: int | None

    def __init__(
        self,
        jdk: JdkEnvironment,
        argv: Iterable[str],
        classpath_entries: Iterable[str],
        input_digest: Digest,
        description: str,
        level: LogLevel = LogLevel.INFO,
        extra_jvm_options: Iterable[str] | None = None,
        extra_nailgun_keys: Iterable[str] | None = None,
        output_files: Iterable[str] | None = None,
        output_directories: Iterable[str] | None = None,
        extra_immutable_input_digests: Mapping[str, Digest] | None = None,
        extra_env: Mapping[str, str] | None = None,
        timeout_seconds: int | float | None = None,
        cache_scope: ProcessCacheScope | None = None,
        use_nailgun: bool = True,
        remote_cache_speculation_delay: int | None = None,
    ):
        object.__setattr__(self, "jdk", jdk)
        object.__setattr__(self, "argv", tuple(argv))
        object.__setattr__(self, "classpath_entries", tuple(classpath_entries))
        object.__setattr__(self, "input_digest", input_digest)
        object.__setattr__(self, "description", description)
        object.__setattr__(self, "level", level)
        object.__setattr__(self, "extra_jvm_options", tuple(extra_jvm_options or ()))
        object.__setattr__(self, "extra_nailgun_keys", tuple(extra_nailgun_keys or ()))
        object.__setattr__(self, "output_files", tuple(output_files or ()))
        object.__setattr__(self, "output_directories", tuple(output_directories or ()))
        object.__setattr__(self, "timeout_seconds", timeout_seconds)
        object.__setattr__(self, "cache_scope", cache_scope)
        object.__setattr__(
            self, "extra_immutable_input_digests", FrozenDict(extra_immutable_input_digests or {})
        )
        object.__setattr__(self, "extra_env", FrozenDict(extra_env or {}))
        object.__setattr__(self, "use_nailgun", use_nailgun)
        object.__setattr__(self, "remote_cache_speculation_delay", remote_cache_speculation_delay)

        self.__post_init__()

    def __post_init__(self):
        if not self.use_nailgun and self.extra_nailgun_keys:
            raise AssertionError(
                "`JvmProcess` specified nailgun keys, but has `use_nailgun=False`. Either "
                "specify `extra_nailgun_keys=None` or `use_nailgun=True`."
            )


_JVM_HEAP_SIZE_UNITS = ["", "k", "m", "g"]


@rule
async def jvm_process(
    bash: BashBinary, request: JvmProcess, jvm: JvmSubsystem, global_options: GlobalOptions
) -> Process:
    jdk = request.jdk

    immutable_input_digests = {
        **jdk.immutable_input_digests,
        **request.extra_immutable_input_digests,
    }
    env = {
        "PANTS_INTERNAL_ABSOLUTE_PREFIX": "",
        **jdk.env,
        **request.extra_env,
    }

    def valid_jvm_opt(opt: str) -> str:
        if opt.startswith("-Xmx"):
            raise ValueError(
                softwrap(
                    f"""
                    Invalid value for JVM options: {opt}.

                    For setting a maximum heap size for the JVM child processes, use
                    `[GLOBAL].process_per_child_memory_usage` option instead.

                    Run `{bin_name()} help-advanced global` for more information.
                    """
                )
            )
        return opt

    max_heap_size = fmt_memory_size(
        global_options.process_per_child_memory_usage, units=_JVM_HEAP_SIZE_UNITS
    )
    jvm_user_options = [*jdk.global_jvm_options, *request.extra_jvm_options]
    jvm_options = [
        f"-Xmx{max_heap_size}",
        *[valid_jvm_opt(opt) for opt in jvm_user_options],
    ]

    use_nailgun = []
    if request.use_nailgun:
        use_nailgun = [*jdk.immutable_input_digests, *request.extra_nailgun_keys]

    remote_cache_speculation_delay_millis = 0
    if request.remote_cache_speculation_delay is not None:
        remote_cache_speculation_delay_millis = request.remote_cache_speculation_delay
    elif request.use_nailgun:
        remote_cache_speculation_delay_millis = jvm.nailgun_remote_cache_speculation_delay

    return Process(
        [*jdk.args(bash, request.classpath_entries), *jvm_options, *request.argv],
        input_digest=request.input_digest,
        immutable_input_digests=immutable_input_digests,
        use_nailgun=use_nailgun,
        description=request.description,
        level=request.level,
        output_directories=request.output_directories,
        env=env,
        timeout_seconds=request.timeout_seconds,
        append_only_caches=jdk.append_only_caches,
        output_files=request.output_files,
        cache_scope=request.cache_scope or ProcessCacheScope.SUCCESSFUL,
        remote_cache_speculation_delay_millis=remote_cache_speculation_delay_millis,
    )


def rules():
    return collect_rules()

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	12✔
5
6	import dataclasses	12✔
7	import logging	12✔
8	import os	12✔
9	import re	12✔
10	import shlex	12✔
11	import textwrap	12✔
12	from collections.abc import Iterable, Mapping	12✔
13	from dataclasses import dataclass	12✔
14	from enum import Enum	12✔
15	from typing import ClassVar	12✔
16
17	from pants.core.environments.target_types import EnvironmentTarget	12✔
18	from pants.core.util_rules.system_binaries import BashBinary, LnBinary	12✔
19	from pants.engine.fs import CreateDigest, Digest, FileContent, FileDigest, MergeDigests	12✔
20	from pants.engine.intrinsics import create_digest, execute_process, merge_digests	12✔
21	from pants.engine.process import Process, ProcessCacheScope	12✔
22	from pants.engine.rules import collect_rules, implicitly, rule	12✔
23	from pants.engine.target import CoarsenedTarget	12✔
24	from pants.jvm.compile import ClasspathEntry	12✔
25	from pants.jvm.resolve.coordinate import Coordinate, Coordinates	12✔
26	from pants.jvm.resolve.coursier_fetch import CoursierLockfileEntry, coursier_fetch_one_coord	12✔
27	from pants.jvm.resolve.coursier_setup import Coursier	12✔
28	from pants.jvm.subsystems import JvmSubsystem	12✔
29	from pants.jvm.target_types import JvmJdkField	12✔
30	from pants.option.global_options import GlobalOptions	12✔
31	from pants.util.docutil import bin_name	12✔
32	from pants.util.frozendict import FrozenDict	12✔
33	from pants.util.logging import LogLevel	12✔
34	from pants.util.meta import classproperty	12✔
35	from pants.util.strutil import fmt_memory_size, softwrap	12✔
36
37	logger = logging.getLogger(__name__)	12✔
38
39
40	@dataclass(frozen=True)	12✔
41	class Nailgun:	12✔
42	classpath_entry: ClasspathEntry	12✔
43
44
45	class DefaultJdk(Enum):	12✔
46	SYSTEM = "system"	12✔
47	SOURCE_DEFAULT = "source_default"	12✔
48
49
50	@dataclass(frozen=True)	12✔
51	class JdkRequest:	12✔
52	"""Request for a JDK with a specific major version, or a default (`--jvm-jdk` or System)."""
53
54	version: str \| DefaultJdk	12✔
55
56	@classproperty	12✔
57	def SYSTEM(cls) -> JdkRequest:	12✔
58	return JdkRequest(DefaultJdk.SYSTEM)	×
59
60	@classproperty	12✔
61	def SOURCE_DEFAULT(cls) -> JdkRequest:	12✔
62	return JdkRequest(DefaultJdk.SOURCE_DEFAULT)	×
63
64	@staticmethod	12✔
65	def from_field(field: JvmJdkField) -> JdkRequest:	12✔
66	version = field.value	×
67	if version == "system":	×
68	return JdkRequest.SYSTEM	×
69	return JdkRequest(version) if version is not None else JdkRequest.SOURCE_DEFAULT	×
70
71	@staticmethod	12✔
72	def from_target(target: CoarsenedTarget) -> JdkRequest:	12✔
73	fields = [t[JvmJdkField] for t in target.members if t.has_field(JvmJdkField)]	×
74
75	if not fields:	×
76	raise ValueError(	×
77	f"Cannot construct a JDK request for {target}, since none of its "
78	f"members have a `{JvmJdkField.alias}=` field:\n{target.bullet_list()}"
79	)
80
81	field = fields[0]	×
82	if not all(f.value == field.value for f in fields):	×
83	values = {f.value for f in fields}	×
84	raise ValueError(	×
85	f"The members of {target} had mismatched values of the "
86	f"`{JvmJdkField.alias}=` field ({values}):\n{target.bullet_list()}"
87	)
88
89	return JdkRequest.from_field(field)	×
90
91
92	@dataclass(frozen=True)	12✔
93	class JdkEnvironment:	12✔
94	_digest: Digest	12✔
95	nailgun_jar: str	12✔
96	coursier: Coursier	12✔
97	jre_major_version: int	12✔
98	global_jvm_options: tuple[str, ...]	12✔
99	java_home_command: str	12✔
100
101	bin_dir: ClassVar[str] = "__jdk"	12✔
102	jdk_preparation_script: ClassVar[str] = f"{bin_dir}/jdk.sh"	12✔
103	java_home: ClassVar[str] = "__java_home"	12✔
104
105	def args(	12✔
106	self, bash: BashBinary, classpath_entries: Iterable[str], chroot: str \| None = None
107	) -> tuple[str, ...]:
108	def in_chroot(path: str) -> str:	×
109	if not chroot:	×
110	return path	×
111	return os.path.join(chroot, path)	×
112
113	return (	×
114	bash.path,
115	in_chroot(self.jdk_preparation_script),
116	f"{self.java_home}/bin/java",
117	"-cp",
118	":".join([in_chroot(self.nailgun_jar), *classpath_entries]),
119	)
120
121	@property	12✔
122	def env(self) -> dict[str, str]:	12✔
123	return self.coursier.env	×
124
125	@property	12✔
126	def append_only_caches(self) -> dict[str, str]:	12✔
127	return self.coursier.append_only_caches	×
128
129	@property	12✔
130	def immutable_input_digests(self) -> dict[str, Digest]:	12✔
131	return {**self.coursier.immutable_input_digests, self.bin_dir: self._digest}	×
132
133
134	@dataclass(frozen=True)	12✔
135	class InternalJdk(JdkEnvironment):	12✔
136	"""The JDK configured for internal Pants usage, rather than for matching source compatibility.
137
138	The InternalJdk should only be used in situations where no classfiles are required for a user's
139	firstparty or thirdparty code (such as for codegen, or analysis of source files).
140	"""
141
142	@classmethod	12✔
143	def from_jdk_environment(cls, env: JdkEnvironment) -> InternalJdk:	12✔
144	return cls(	×
145	env._digest,
146	env.nailgun_jar,
147	env.coursier,
148	env.jre_major_version,
149	env.global_jvm_options,
150	env.java_home_command,
151	)
152
153
154	VERSION_REGEX = re.compile(r"version \"(.+?)\"")	12✔
155
156
157	def parse_jre_major_version(version_lines: str) -> int \| None:	12✔
158	for line in version_lines.splitlines():	1✔
159	m = VERSION_REGEX.search(line)	1✔
160	if m:	1✔
161	major_version, _, _ = m[1].partition(".")	1✔
162	return int(major_version)	1✔
163	return None	×
164
165
166	@rule	12✔
167	async def fetch_nailgun() -> Nailgun:	12✔
168	nailgun = await coursier_fetch_one_coord(	×
169	CoursierLockfileEntry(
170	coord=Coordinate.from_coord_str("com.martiansoftware:nailgun-server:0.9.1"),
171	file_name="com.martiansoftware_nailgun-server_0.9.1.jar",
172	direct_dependencies=Coordinates(),
173	dependencies=Coordinates(),
174	file_digest=FileDigest(
175	fingerprint="4518faa6bf4bd26fccdc4d85e1625dc679381a08d56872d8ad12151dda9cef25",
176	serialized_bytes_length=32927,
177	),
178	)
179	)
180
181	return Nailgun(nailgun)	×
182
183
184	@rule	12✔
185	async def prepare_jdk_environment(	12✔
186	jvm: JvmSubsystem,
187	jvm_env_aware: JvmSubsystem.EnvironmentAware,
188	coursier: Coursier,
189	nailgun_: Nailgun,
190	bash: BashBinary,
191	ln: LnBinary,
192	request: JdkRequest,
193	env_target: EnvironmentTarget,
194	) -> JdkEnvironment:
195	nailgun = nailgun_.classpath_entry	×
196
197	version = request.version	×
198	if version == DefaultJdk.SOURCE_DEFAULT:	×
199	version = jvm.jdk	×
200
201	# TODO: add support for system JDKs with specific version
202	if version is DefaultJdk.SYSTEM:	×
203	coursier_jdk_option = "--system-jvm"	×
204	else:
205	coursier_jdk_option = f"--jvm={version}"	×
206
207	if not coursier.jvm_index:	×
208	coursier_options = ["java-home", coursier_jdk_option]	×
209	else:
210	jvm_index_option = f"--jvm-index={coursier.jvm_index}"	×
211	coursier_options = ["java-home", jvm_index_option, coursier_jdk_option]	×
212
213	# TODO(#16104) This argument re-writing code should use the native {chroot} support.
214	# See also `run` for other argument re-writing code.
215	def prefixed(arg: str) -> str:	×
216	quoted = shlex.quote(arg)	×
217	if arg.startswith("__"):	×
218	return f"${{PANTS_INTERNAL_ABSOLUTE_PREFIX}}{quoted}"	×
219	else:
220	return quoted	×
221
222	optionally_prefixed_coursier_args = [prefixed(arg) for arg in coursier.args(coursier_options)]	×
223	# NB: We `set +e` in the subshell to ensure that it exits as well.
224	# see https://unix.stackexchange.com/a/23099
225	java_home_command = " ".join(("set +e;", *optionally_prefixed_coursier_args))	×
226
227	env = {	×
228	"PANTS_INTERNAL_ABSOLUTE_PREFIX": "",
229	**coursier.env,
230	}
231
232	java_version_result = await execute_process(	×
233	Process(
234	argv=(
235	bash.path,
236	"-c",
237	f"$({java_home_command})/bin/java -version",
238	),
239	append_only_caches=coursier.append_only_caches,
240	immutable_input_digests=coursier.immutable_input_digests,
241	env=env,
242	description=f"Ensure download of JDK {coursier_jdk_option}.",
243	cache_scope=env_target.executable_search_path_cache_scope(),
244	level=LogLevel.DEBUG,
245	),
246	**implicitly(),
247	)
248
249	if java_version_result.exit_code != 0:	×
250	raise ValueError(	×
251	f"Failed to locate Java for JDK `{version}`:\n"
252	f"{java_version_result.stderr.decode('utf-8')}"
253	)
254
NEW 255	java_version_raw = java_version_result.stderr.decode("utf-8").strip()	×
256
257	# Filter out Coursier download progress messages which are non-deterministic
258	# and would cause cache key instability. Keep only actual java -version output.
NEW UNCOV 259	java_version_lines = []	×
NEW UNCOV 260	for line in java_version_raw.splitlines():	×
NEW UNCOV 261	line_lower = line.lower().strip()	×
262	# Skip Coursier download progress messages
NEW UNCOV 263	if any(skip in line_lower for skip in [	×
264	"downloading",
265	"still downloading",
266	"downloaded",
267	"https://",
268	"http://",
269	"found ",
270	]):
NEW UNCOV 271	continue	×
272	# Keep lines that look like java -version output
NEW UNCOV 273	if line.strip():	×
NEW UNCOV 274	java_version_lines.append(line)	×
275
NEW UNCOV 276	java_version = "\n".join(java_version_lines)	×
UNCOV 277	jre_major_version = parse_jre_major_version(java_version)	×
278	if not jre_major_version:	×
UNCOV 279	raise ValueError(	×
280	"Pants was unable to parse the output of `java -version` for JDK "
281	f"`{request.version}`. Please open an issue at "
282	"https://github.com/pantsbuild/pants/issues/new/choose with the following output:\n\n"
283	f"{java_version}"
284	)
285
286	# TODO: Locate `ln`.
UNCOV 287	version_comment = "\n".join(f"# {line}" for line in java_version.splitlines())	×
UNCOV 288	ln_path = shlex.quote(ln.path)	×
UNCOV 289	jdk_preparation_script = textwrap.dedent( # noqa: PNT20	×
290	f"""\
291	# pants javac script using Coursier {coursier_jdk_option}. `java -version`:"
292	{version_comment}
293	set -eu
294
295	{ln_path} -s "$({java_home_command})" "${{PANTS_INTERNAL_ABSOLUTE_PREFIX}}{JdkEnvironment.java_home}"
296	exec "$@"
297	"""
298	)
UNCOV 299	jdk_preparation_script_digest = await create_digest(	×
300	CreateDigest(
301	[
302	FileContent(
303	os.path.basename(JdkEnvironment.jdk_preparation_script),
304	jdk_preparation_script.encode("utf-8"),
305	is_executable=True,
306	),
307	]
308	),
309	)
310
UNCOV 311	return JdkEnvironment(	×
312	_digest=await merge_digests(
313	MergeDigests(
314	[
315	jdk_preparation_script_digest,
316	nailgun.digest,
317	]
318	),
319	),
320	global_jvm_options=jvm_env_aware.global_options,
321	nailgun_jar=os.path.join(JdkEnvironment.bin_dir, nailgun.filenames[0]),
322	coursier=coursier,
323	jre_major_version=jre_major_version,
324	java_home_command=java_home_command,
325	)
326
327
328	@rule	12✔
329	async def internal_jdk(jvm: JvmSubsystem) -> InternalJdk:	12✔
330	"""Creates a `JdkEnvironment` object based on the JVM subsystem options.
331
332	This is used for providing a predictable JDK version for Pants' internal usage rather than for
333	matching compatibility with source files (e.g. compilation/testing).
334	"""
335
UNCOV 336	request = JdkRequest(jvm.tool_jdk) if jvm.tool_jdk is not None else JdkRequest.SYSTEM	×
UNCOV 337	env = await prepare_jdk_environment(**implicitly({request: JdkRequest}))	×
UNCOV 338	return InternalJdk.from_jdk_environment(env)	×
339
340
341	@dataclass(frozen=True)	12✔
342	class JvmProcess:	12✔
343	jdk: JdkEnvironment	12✔
344	argv: tuple[str, ...]	12✔
345	classpath_entries: tuple[str, ...]	12✔
346	input_digest: Digest	12✔
347	description: str = dataclasses.field(compare=False)	12✔
348	level: LogLevel	12✔
349	extra_jvm_options: tuple[str, ...]	12✔
350	extra_nailgun_keys: tuple[str, ...]	12✔
351	output_files: tuple[str, ...]	12✔
352	output_directories: tuple[str, ...]	12✔
353	timeout_seconds: int \| float \| None	12✔
354	extra_immutable_input_digests: FrozenDict[str, Digest]	12✔
355	extra_env: FrozenDict[str, str]	12✔
356	cache_scope: ProcessCacheScope \| None	12✔
357	use_nailgun: bool	12✔
358	remote_cache_speculation_delay: int \| None	12✔
359
360	def __init__(	12✔
361	self,
362	jdk: JdkEnvironment,
363	argv: Iterable[str],
364	classpath_entries: Iterable[str],
365	input_digest: Digest,
366	description: str,
367	level: LogLevel = LogLevel.INFO,
368	extra_jvm_options: Iterable[str] \| None = None,
369	extra_nailgun_keys: Iterable[str] \| None = None,
370	output_files: Iterable[str] \| None = None,
371	output_directories: Iterable[str] \| None = None,
372	extra_immutable_input_digests: Mapping[str, Digest] \| None = None,
373	extra_env: Mapping[str, str] \| None = None,
374	timeout_seconds: int \| float \| None = None,
375	cache_scope: ProcessCacheScope \| None = None,
376	use_nailgun: bool = True,
377	remote_cache_speculation_delay: int \| None = None,
378	):
379	object.__setattr__(self, "jdk", jdk)	2✔
380	object.__setattr__(self, "argv", tuple(argv))	2✔
381	object.__setattr__(self, "classpath_entries", tuple(classpath_entries))	2✔
382	object.__setattr__(self, "input_digest", input_digest)	2✔
383	object.__setattr__(self, "description", description)	2✔
384	object.__setattr__(self, "level", level)	2✔
385	object.__setattr__(self, "extra_jvm_options", tuple(extra_jvm_options or ()))	2✔
386	object.__setattr__(self, "extra_nailgun_keys", tuple(extra_nailgun_keys or ()))	2✔
387	object.__setattr__(self, "output_files", tuple(output_files or ()))	2✔
388	object.__setattr__(self, "output_directories", tuple(output_directories or ()))	2✔
389	object.__setattr__(self, "timeout_seconds", timeout_seconds)	2✔
390	object.__setattr__(self, "cache_scope", cache_scope)	2✔
391	object.__setattr__(	2✔
392	self, "extra_immutable_input_digests", FrozenDict(extra_immutable_input_digests or {})
393	)
394	object.__setattr__(self, "extra_env", FrozenDict(extra_env or {}))	2✔
395	object.__setattr__(self, "use_nailgun", use_nailgun)	2✔
396	object.__setattr__(self, "remote_cache_speculation_delay", remote_cache_speculation_delay)	2✔
397
398	self.__post_init__()	2✔
399
400	def __post_init__(self):	12✔
401	if not self.use_nailgun and self.extra_nailgun_keys:	2✔
UNCOV 402	raise AssertionError(	×
403	"`JvmProcess` specified nailgun keys, but has `use_nailgun=False`. Either "
404	"specify `extra_nailgun_keys=None` or `use_nailgun=True`."
405	)
406
407
408	_JVM_HEAP_SIZE_UNITS = ["", "k", "m", "g"]	12✔
409
410
411	@rule	12✔
412	async def jvm_process(	12✔
413	bash: BashBinary, request: JvmProcess, jvm: JvmSubsystem, global_options: GlobalOptions
414	) -> Process:
UNCOV 415	jdk = request.jdk	×
416
UNCOV 417	immutable_input_digests = {	×
418	**jdk.immutable_input_digests,
419	**request.extra_immutable_input_digests,
420	}
UNCOV 421	env = {	×
422	"PANTS_INTERNAL_ABSOLUTE_PREFIX": "",
423	**jdk.env,
424	**request.extra_env,
425	}
426
UNCOV 427	def valid_jvm_opt(opt: str) -> str:	×
UNCOV 428	if opt.startswith("-Xmx"):	×
UNCOV 429	raise ValueError(	×
430	softwrap(
431	f"""
432	Invalid value for JVM options: {opt}.
433
434	For setting a maximum heap size for the JVM child processes, use
435	`[GLOBAL].process_per_child_memory_usage` option instead.
436
437	Run `{bin_name()} help-advanced global` for more information.
438	"""
439	)
440	)
441	return opt	×
442
UNCOV 443	max_heap_size = fmt_memory_size(	×
444	global_options.process_per_child_memory_usage, units=_JVM_HEAP_SIZE_UNITS
445	)
UNCOV 446	jvm_user_options = [jdk.global_jvm_options, request.extra_jvm_options]	×
UNCOV 447	jvm_options = [	×
448	f"-Xmx{max_heap_size}",
449	*[valid_jvm_opt(opt) for opt in jvm_user_options],
450	]
451
UNCOV 452	use_nailgun = []	×
UNCOV 453	if request.use_nailgun:	×
UNCOV 454	use_nailgun = [jdk.immutable_input_digests, request.extra_nailgun_keys]	×
455
UNCOV 456	remote_cache_speculation_delay_millis = 0	×
UNCOV 457	if request.remote_cache_speculation_delay is not None:	×
UNCOV 458	remote_cache_speculation_delay_millis = request.remote_cache_speculation_delay	×
UNCOV 459	elif request.use_nailgun:	×
UNCOV 460	remote_cache_speculation_delay_millis = jvm.nailgun_remote_cache_speculation_delay	×
461
UNCOV 462	return Process(	×
463	[jdk.args(bash, request.classpath_entries), jvm_options, *request.argv],
464	input_digest=request.input_digest,
465	immutable_input_digests=immutable_input_digests,
466	use_nailgun=use_nailgun,
467	description=request.description,
468	level=request.level,
469	output_directories=request.output_directories,
470	env=env,
471	timeout_seconds=request.timeout_seconds,
472	append_only_caches=jdk.append_only_caches,
473	output_files=request.output_files,
474	cache_scope=request.cache_scope or ProcessCacheScope.SUCCESSFUL,
475	remote_cache_speculation_delay_millis=remote_cache_speculation_delay_millis,
476	)
477
478
479	def rules():	12✔
480	return collect_rules()	12✔

pantsbuild / pants / 21365507481

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous