20927636681

Committed 12 Jan 2026 04:51PM UTC coverage: 80.275%. Remained the same

Build # 20927636681

Build Type

push

github

Committed by

web-flow

Commit Message

Fix incorrect use of `PEX_PYTHON`. (#22994)

This was established in #18433 and only worked due to a Pex bug that
unconditionally stripped `PEX_PYTHON` on PEX boot. the portions of 
#18433 related to the cleanup so described are reverted:

> Additionally when making this change, the Python/pex code was
> refactored so that we always use this Python to run pex, with Python
> either being chosen by pex, or by using PEX_PYTHON env var at
> runtime. I think this is a nice cleanup of the handshake between
> CompletePexEnvironment.create_argv and
> CompletePexEnvironment.environment_dict used to have.

Fixes #22988

Run Details

3 of 11 new or added lines in 6 files covered. (27.27%)

1 existing line in 1 file now uncovered.

78798 of 98160 relevant lines covered (80.28%)

3.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.0

/src/python/pants/backend/python/goals/run_helper.py

# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
from __future__ import annotations

import dataclasses
import os
import textwrap
from collections.abc import Iterable

from pants.backend.python.subsystems.debugpy import DebugPy
from pants.backend.python.target_types import (
    ConsoleScript,
    Executable,
    PexEntryPointField,
    ResolvePexEntryPointRequest,
)
from pants.backend.python.target_types_rules import resolve_pex_entry_point
from pants.backend.python.util_rules.pex import (
    Pex,
    VenvPexRequest,
    create_venv_pex,
)
from pants.backend.python.util_rules.pex_environment import PexEnvironment
from pants.backend.python.util_rules.pex_from_targets import (
    PexFromTargetsRequest,
    create_pex_from_targets,
)
from pants.backend.python.util_rules.python_sources import (
    PythonSourceFilesRequest,
    prepare_python_sources,
)
from pants.core.goals.run import RunDebugAdapterRequest, RunRequest
from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem
from pants.engine.addresses import Address
from pants.engine.fs import CreateDigest, FileContent, MergeDigests
from pants.engine.internals.graph import transitive_targets as transitive_targets_get
from pants.engine.intrinsics import create_digest, merge_digests
from pants.engine.rules import concurrently, implicitly
from pants.engine.target import TransitiveTargetsRequest
from pants.util.frozendict import FrozenDict


def _in_chroot(relpath: str) -> str:
    return os.path.join("{chroot}", relpath)


async def _create_python_source_run_request(
    address: Address,
    *,
    entry_point_field: PexEntryPointField,
    pex_env: PexEnvironment,
    run_in_sandbox: bool,
    pex_path: Iterable[Pex] = (),
    console_script: ConsoleScript | None = None,
    executable: Executable | None = None,
) -> RunRequest:
    addresses = [address]
    entry_point, transitive_targets = await concurrently(
        resolve_pex_entry_point(ResolvePexEntryPointRequest(entry_point_field)),
        transitive_targets_get(TransitiveTargetsRequest(addresses), **implicitly()),
    )

    pex_filename = (
        address.generated_name.replace(".", "_") if address.generated_name else address.target_name
    )

    pex_request, sources = await concurrently(
        create_pex_from_targets(
            PexFromTargetsRequest(
                addresses,
                output_filename=f"{pex_filename}.pex",
                internal_only=True,
                include_source_files=False,
                include_local_dists=True,
                # `PEX_EXTRA_SYS_PATH` should contain this entry_point's module.
                main=executable or console_script or entry_point.val,
                additional_args=(
                    # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
                    # below, it's important for any app that re-executes itself that these environment
                    # variables are not stripped.
                    "--no-strip-pex-env",
                ),
            ),
            **implicitly(),
        ),
        prepare_python_sources(
            PythonSourceFilesRequest(transitive_targets.closure, include_files=True), **implicitly()
        ),
    )

    pex_request = dataclasses.replace(pex_request, pex_path=(*pex_request.pex_path, *pex_path))

    if run_in_sandbox:
        # Note that a RunRequest always expects to run directly in the sandbox/workspace
        # root, hence working_directory=None.
        complete_pex_environment = pex_env.in_sandbox(working_directory=None)
    else:
        complete_pex_environment = pex_env.in_workspace()
    venv_pex = await create_venv_pex(
        VenvPexRequest(pex_request, complete_pex_environment), **implicitly()
    )
    input_digests = [
        venv_pex.digest,
        # Note regarding not-in-sandbox mode: You might think that the sources don't need to be copied
        # into the chroot when using inline sources. But they do, because some of them might be
        # codegenned, and those won't exist in the inline source tree. Rather than incurring the
        # complexity of figuring out here which sources were codegenned, we copy everything.
        # The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline
        # sources will take precedence and their copies in the chroot will be ignored.
        sources.source_files.snapshot.digest,
    ]
    merged_digest = await merge_digests(MergeDigests(input_digests))

    chrooted_source_roots = [_in_chroot(sr) for sr in sources.source_roots]
    # The order here is important: we want the in-repo sources to take precedence over their
    # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
    source_roots = [
        *([] if run_in_sandbox else sources.source_roots),
        *chrooted_source_roots,
    ]
    extra_env = {
        **complete_pex_environment.environment_dict(python_configured=True),
        "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots),
    }
    append_only_caches = (
        FrozenDict({}) if venv_pex.append_only_caches is None else venv_pex.append_only_caches
    )

    return RunRequest(
        digest=merged_digest,
        args=[_in_chroot(venv_pex.pex.argv0)],
        extra_env=extra_env,
        append_only_caches={
            **complete_pex_environment.append_only_caches,
            **append_only_caches,
        },
    )


async def _create_python_source_run_dap_request(
    regular_run_request: RunRequest,
    *,
    debugpy: DebugPy,
    debug_adapter: DebugAdapterSubsystem,
    run_in_sandbox: bool,
) -> RunDebugAdapterRequest:
    launcher_digest = await create_digest(
        CreateDigest(
            [
                FileContent(
                    "__debugpy_launcher.py",
                    textwrap.dedent(
                        """
                            import os
                            CHROOT = os.environ["PANTS_CHROOT"]

                            del os.environ["PEX_INTERPRETER"]

                            import debugpy._vendored.force_pydevd
                            from _pydevd_bundle.pydevd_process_net_command_json import PyDevJsonCommandProcessor
                            orig_resolve_remote_root = PyDevJsonCommandProcessor._resolve_remote_root

                            def patched_resolve_remote_root(self, local_root, remote_root):
                                if remote_root == ".":
                                    remote_root = CHROOT
                                return orig_resolve_remote_root(self, local_root, remote_root)

                            PyDevJsonCommandProcessor._resolve_remote_root = patched_resolve_remote_root

                            from debugpy.server import cli
                            cli.main()
                            """
                    ).encode("utf-8"),
                ),
            ]
        ),
    )

    merged_digest = await merge_digests(MergeDigests([regular_run_request.digest, launcher_digest]))
    extra_env = dict(regular_run_request.extra_env)
    extra_env["PEX_INTERPRETER"] = "1"
    # See https://github.com/pantsbuild/pants/issues/17540
    # and https://github.com/pantsbuild/pants/issues/18243
    # For `run --debug-adapter`, the client might send a `pathMappings`
    # (this is likely as VS Code likes to configure that by default) with a `remoteRoot` of ".".
    #
    # For `run`, CWD is the build root. If `run_in_sandbox` is False, everything is OK.
    # If `run_in_sandbox` is True, breakpoints won't be hit as CWD != sandbox root.
    #
    # We fix this by monkeypatching pydevd (the library powering debugpy) so that a remoteRoot of "."
    # means the sandbox root.
    # See https://github.com/fabioz/PyDev.Debugger/pull/243 for a better solution.
    extra_env["PANTS_CHROOT"] = _in_chroot("").rstrip("/") if run_in_sandbox else "."
    args = [
        *regular_run_request.args,
        _in_chroot("__debugpy_launcher.py"),
        *debugpy.get_args(debug_adapter),
    ]

    return RunDebugAdapterRequest(
        digest=merged_digest,
        args=args,
        extra_env=extra_env,
        append_only_caches=regular_run_request.append_only_caches,
        immutable_input_digests=regular_run_request.immutable_input_digests,
    )

1	# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	from __future__ import annotations	9✔
4
5	import dataclasses	9✔
6	import os	9✔
7	import textwrap	9✔
8	from collections.abc import Iterable	9✔
9
10	from pants.backend.python.subsystems.debugpy import DebugPy	9✔
11	from pants.backend.python.target_types import (	9✔
12	ConsoleScript,
13	Executable,
14	PexEntryPointField,
15	ResolvePexEntryPointRequest,
16	)
17	from pants.backend.python.target_types_rules import resolve_pex_entry_point	9✔
18	from pants.backend.python.util_rules.pex import (	9✔
19	Pex,
20	VenvPexRequest,
21	create_venv_pex,
22	)
23	from pants.backend.python.util_rules.pex_environment import PexEnvironment	9✔
24	from pants.backend.python.util_rules.pex_from_targets import (	9✔
25	PexFromTargetsRequest,
26	create_pex_from_targets,
27	)
28	from pants.backend.python.util_rules.python_sources import (	9✔
29	PythonSourceFilesRequest,
30	prepare_python_sources,
31	)
32	from pants.core.goals.run import RunDebugAdapterRequest, RunRequest	9✔
33	from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem	9✔
34	from pants.engine.addresses import Address	9✔
35	from pants.engine.fs import CreateDigest, FileContent, MergeDigests	9✔
36	from pants.engine.internals.graph import transitive_targets as transitive_targets_get	9✔
37	from pants.engine.intrinsics import create_digest, merge_digests	9✔
38	from pants.engine.rules import concurrently, implicitly	9✔
39	from pants.engine.target import TransitiveTargetsRequest	9✔
40	from pants.util.frozendict import FrozenDict	9✔
41
42
43	def _in_chroot(relpath: str) -> str:	9✔
44	return os.path.join("{chroot}", relpath)	×
45
46
47	async def _create_python_source_run_request(	9✔
48	address: Address,
49	*,
50	entry_point_field: PexEntryPointField,
51	pex_env: PexEnvironment,
52	run_in_sandbox: bool,
53	pex_path: Iterable[Pex] = (),
54	console_script: ConsoleScript \| None = None,
55	executable: Executable \| None = None,
56	) -> RunRequest:
57	addresses = [address]	×
58	entry_point, transitive_targets = await concurrently(	×
59	resolve_pex_entry_point(ResolvePexEntryPointRequest(entry_point_field)),
60	transitive_targets_get(TransitiveTargetsRequest(addresses), **implicitly()),
61	)
62
63	pex_filename = (	×
64	address.generated_name.replace(".", "_") if address.generated_name else address.target_name
65	)
66
67	pex_request, sources = await concurrently(	×
68	create_pex_from_targets(
69	PexFromTargetsRequest(
70	addresses,
71	output_filename=f"{pex_filename}.pex",
72	internal_only=True,
73	include_source_files=False,
74	include_local_dists=True,
75	# `PEX_EXTRA_SYS_PATH` should contain this entry_point's module.
76	main=executable or console_script or entry_point.val,
77	additional_args=(
78	# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
79	# below, it's important for any app that re-executes itself that these environment
80	# variables are not stripped.
81	"--no-strip-pex-env",
82	),
83	),
84	**implicitly(),
85	),
86	prepare_python_sources(
87	PythonSourceFilesRequest(transitive_targets.closure, include_files=True), **implicitly()
88	),
89	)
90
91	pex_request = dataclasses.replace(pex_request, pex_path=(pex_request.pex_path, pex_path))	×
92
93	if run_in_sandbox:	×
94	# Note that a RunRequest always expects to run directly in the sandbox/workspace
95	# root, hence working_directory=None.
96	complete_pex_environment = pex_env.in_sandbox(working_directory=None)	×
97	else:
98	complete_pex_environment = pex_env.in_workspace()	×
NEW 99	venv_pex = await create_venv_pex(	×
100	VenvPexRequest(pex_request, complete_pex_environment), **implicitly()
101	)
102	input_digests = [	×
103	venv_pex.digest,
104	# Note regarding not-in-sandbox mode: You might think that the sources don't need to be copied
105	# into the chroot when using inline sources. But they do, because some of them might be
106	# codegenned, and those won't exist in the inline source tree. Rather than incurring the
107	# complexity of figuring out here which sources were codegenned, we copy everything.
108	# The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline
109	# sources will take precedence and their copies in the chroot will be ignored.
110	sources.source_files.snapshot.digest,
111	]
112	merged_digest = await merge_digests(MergeDigests(input_digests))	×
113
114	chrooted_source_roots = [_in_chroot(sr) for sr in sources.source_roots]	×
115	# The order here is important: we want the in-repo sources to take precedence over their
116	# copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
117	source_roots = [	×
118	*([] if run_in_sandbox else sources.source_roots),
119	*chrooted_source_roots,
120	]
121	extra_env = {	×
122	**complete_pex_environment.environment_dict(python_configured=True),
123	"PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots),
124	}
125	append_only_caches = (	×
126	FrozenDict({}) if venv_pex.append_only_caches is None else venv_pex.append_only_caches
127	)
128
129	return RunRequest(	×
130	digest=merged_digest,
131	args=[_in_chroot(venv_pex.pex.argv0)],
132	extra_env=extra_env,
133	append_only_caches={
134	**complete_pex_environment.append_only_caches,
135	**append_only_caches,
136	},
137	)
138
139
140	async def _create_python_source_run_dap_request(	9✔
141	regular_run_request: RunRequest,
142	*,
143	debugpy: DebugPy,
144	debug_adapter: DebugAdapterSubsystem,
145	run_in_sandbox: bool,
146	) -> RunDebugAdapterRequest:
147	launcher_digest = await create_digest(	×
148	CreateDigest(
149	[
150	FileContent(
151	"__debugpy_launcher.py",
152	textwrap.dedent(
153	"""
154	import os
155	CHROOT = os.environ["PANTS_CHROOT"]
156
157	del os.environ["PEX_INTERPRETER"]
158
159	import debugpy._vendored.force_pydevd
160	from _pydevd_bundle.pydevd_process_net_command_json import PyDevJsonCommandProcessor
161	orig_resolve_remote_root = PyDevJsonCommandProcessor._resolve_remote_root
162
163	def patched_resolve_remote_root(self, local_root, remote_root):
164	if remote_root == ".":
165	remote_root = CHROOT
166	return orig_resolve_remote_root(self, local_root, remote_root)
167
168	PyDevJsonCommandProcessor._resolve_remote_root = patched_resolve_remote_root
169
170	from debugpy.server import cli
171	cli.main()
172	"""
173	).encode("utf-8"),
174	),
175	]
176	),
177	)
178
179	merged_digest = await merge_digests(MergeDigests([regular_run_request.digest, launcher_digest]))	×
180	extra_env = dict(regular_run_request.extra_env)	×
181	extra_env["PEX_INTERPRETER"] = "1"	×
182	# See https://github.com/pantsbuild/pants/issues/17540
183	# and https://github.com/pantsbuild/pants/issues/18243
184	# For `run --debug-adapter`, the client might send a `pathMappings`
185	# (this is likely as VS Code likes to configure that by default) with a `remoteRoot` of ".".
186	#
187	# For `run`, CWD is the build root. If `run_in_sandbox` is False, everything is OK.
188	# If `run_in_sandbox` is True, breakpoints won't be hit as CWD != sandbox root.
189	#
190	# We fix this by monkeypatching pydevd (the library powering debugpy) so that a remoteRoot of "."
191	# means the sandbox root.
192	# See https://github.com/fabioz/PyDev.Debugger/pull/243 for a better solution.
193	extra_env["PANTS_CHROOT"] = _in_chroot("").rstrip("/") if run_in_sandbox else "."	×
194	args = [	×
195	*regular_run_request.args,
196	_in_chroot("__debugpy_launcher.py"),
197	*debugpy.get_args(debug_adapter),
198	]
199
200	return RunDebugAdapterRequest(	×
201	digest=merged_digest,
202	args=args,
203	extra_env=extra_env,
204	append_only_caches=regular_run_request.append_only_caches,
205	immutable_input_digests=regular_run_request.immutable_input_digests,
206	)

pantsbuild / pants / 20927636681

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous