20937871815

Committed 12 Jan 2026 10:54PM UTC coverage: 80.279% (-0.003%) from 80.282%

Build # 20937871815

Build Type

push

github

Committed by

web-flow

Commit Message

Manual Cherry Pick: Fix incorrect use of `PEX_PYTHON`. (#22994) (#23004)

This was established in #18433 and only worked due to a Pex bug that
unconditionally stripped `PEX_PYTHON` on PEX boot. the portions of
#18433 related to the cleanup so described are reverted:

> Additionally when making this change, the Python/pex code was
> refactored so that we always use this Python to run pex, with Python
> either being chosen by pex, or by using PEX_PYTHON env var at
> runtime. I think this is a nice cleanup of the handshake between
> CompletePexEnvironment.create_argv and
> CompletePexEnvironment.environment_dict used to have.

Fixes #22988

---------

Co-authored-by: John Sirois <john.sirois@gmail.com>

Run Details

4 of 12 new or added lines in 6 files covered. (33.33%)

3 existing lines in 2 files now uncovered.

77969 of 97122 relevant lines covered (80.28%)

3.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.0

/src/python/pants/backend/python/goals/run_helper.py

# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
from __future__ import annotations

import dataclasses
import os
import textwrap
from collections.abc import Iterable

from pants.backend.python.subsystems.debugpy import DebugPy
from pants.backend.python.target_types import (
    ConsoleScript,
    Executable,
    PexEntryPointField,
    ResolvePexEntryPointRequest,
)
from pants.backend.python.target_types_rules import resolve_pex_entry_point
from pants.backend.python.util_rules.pex import Pex, VenvPexRequest, create_venv_pex
from pants.backend.python.util_rules.pex_environment import PexEnvironment
from pants.backend.python.util_rules.pex_from_targets import (
    PexFromTargetsRequest,
    create_pex_from_targets,
)
from pants.backend.python.util_rules.python_sources import (
    PythonSourceFilesRequest,
    prepare_python_sources,
)
from pants.core.goals.run import RunDebugAdapterRequest, RunRequest
from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem
from pants.engine.addresses import Address
from pants.engine.fs import CreateDigest, FileContent, MergeDigests
from pants.engine.internals.graph import transitive_targets as transitive_targets_get
from pants.engine.intrinsics import create_digest, merge_digests
from pants.engine.rules import concurrently, implicitly
from pants.engine.target import TransitiveTargetsRequest
from pants.util.frozendict import FrozenDict


def _in_chroot(relpath: str) -> str:
    return os.path.join("{chroot}", relpath)


async def _create_python_source_run_request(
    address: Address,
    *,
    entry_point_field: PexEntryPointField,
    pex_env: PexEnvironment,
    run_in_sandbox: bool,
    pex_path: Iterable[Pex] = (),
    console_script: ConsoleScript | None = None,
    executable: Executable | None = None,
) -> RunRequest:
    addresses = [address]
    entry_point, transitive_targets = await concurrently(
        resolve_pex_entry_point(ResolvePexEntryPointRequest(entry_point_field)),
        transitive_targets_get(TransitiveTargetsRequest(addresses), **implicitly()),
    )

    pex_filename = (
        address.generated_name.replace(".", "_") if address.generated_name else address.target_name
    )

    pex_request, sources = await concurrently(
        create_pex_from_targets(
            PexFromTargetsRequest(
                addresses,
                output_filename=f"{pex_filename}.pex",
                internal_only=True,
                include_source_files=False,
                include_local_dists=True,
                # `PEX_EXTRA_SYS_PATH` should contain this entry_point's module.
                main=executable or console_script or entry_point.val,
                additional_args=(
                    # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
                    # below, it's important for any app that re-executes itself that these environment
                    # variables are not stripped.
                    "--no-strip-pex-env",
                ),
            ),
            **implicitly(),
        ),
        prepare_python_sources(
            PythonSourceFilesRequest(transitive_targets.closure, include_files=True), **implicitly()
        ),
    )

    pex_request = dataclasses.replace(pex_request, pex_path=(*pex_request.pex_path, *pex_path))

    if run_in_sandbox:
        # Note that a RunRequest always expects to run directly in the sandbox/workspace
        # root, hence working_directory=None.
        complete_pex_environment = pex_env.in_sandbox(working_directory=None)
    else:
        complete_pex_environment = pex_env.in_workspace()
    venv_pex = await create_venv_pex(
        VenvPexRequest(pex_request, complete_pex_environment), **implicitly()
    )
    input_digests = [
        venv_pex.digest,
        # Note regarding not-in-sandbox mode: You might think that the sources don't need to be copied
        # into the chroot when using inline sources. But they do, because some of them might be
        # codegenned, and those won't exist in the inline source tree. Rather than incurring the
        # complexity of figuring out here which sources were codegenned, we copy everything.
        # The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline
        # sources will take precedence and their copies in the chroot will be ignored.
        sources.source_files.snapshot.digest,
    ]
    merged_digest = await merge_digests(MergeDigests(input_digests))

    chrooted_source_roots = [_in_chroot(sr) for sr in sources.source_roots]
    # The order here is important: we want the in-repo sources to take precedence over their
    # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
    source_roots = [
        *([] if run_in_sandbox else sources.source_roots),
        *chrooted_source_roots,
    ]
    extra_env = {
        **complete_pex_environment.environment_dict(python_configured=True),
        "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots),
    }
    append_only_caches = (
        FrozenDict({}) if venv_pex.append_only_caches is None else venv_pex.append_only_caches
    )

    return RunRequest(
        digest=merged_digest,
        args=[_in_chroot(venv_pex.pex.argv0)],
        extra_env=extra_env,
        append_only_caches={
            **complete_pex_environment.append_only_caches,
            **append_only_caches,
        },
    )


async def _create_python_source_run_dap_request(
    regular_run_request: RunRequest,
    *,
    debugpy: DebugPy,
    debug_adapter: DebugAdapterSubsystem,
    run_in_sandbox: bool,
) -> RunDebugAdapterRequest:
    launcher_digest = await create_digest(
        CreateDigest(
            [
                FileContent(
                    "__debugpy_launcher.py",
                    textwrap.dedent(
                        """
                            import os
                            CHROOT = os.environ["PANTS_CHROOT"]

                            del os.environ["PEX_INTERPRETER"]

                            import debugpy._vendored.force_pydevd
                            from _pydevd_bundle.pydevd_process_net_command_json import PyDevJsonCommandProcessor
                            orig_resolve_remote_root = PyDevJsonCommandProcessor._resolve_remote_root

                            def patched_resolve_remote_root(self, local_root, remote_root):
                                if remote_root == ".":
                                    remote_root = CHROOT
                                return orig_resolve_remote_root(self, local_root, remote_root)

                            PyDevJsonCommandProcessor._resolve_remote_root = patched_resolve_remote_root

                            from debugpy.server import cli
                            cli.main()
                            """
                    ).encode("utf-8"),
                ),
            ]
        ),
    )

    merged_digest = await merge_digests(MergeDigests([regular_run_request.digest, launcher_digest]))
    extra_env = dict(regular_run_request.extra_env)
    extra_env["PEX_INTERPRETER"] = "1"
    # See https://github.com/pantsbuild/pants/issues/17540
    # and https://github.com/pantsbuild/pants/issues/18243
    # For `run --debug-adapter`, the client might send a `pathMappings`
    # (this is likely as VS Code likes to configure that by default) with a `remoteRoot` of ".".
    #
    # For `run`, CWD is the build root. If `run_in_sandbox` is False, everything is OK.
    # If `run_in_sandbox` is True, breakpoints won't be hit as CWD != sandbox root.
    #
    # We fix this by monkeypatching pydevd (the library powering debugpy) so that a remoteRoot of "."
    # means the sandbox root.
    # See https://github.com/fabioz/PyDev.Debugger/pull/243 for a better solution.
    extra_env["PANTS_CHROOT"] = _in_chroot("").rstrip("/") if run_in_sandbox else "."
    args = [
        *regular_run_request.args,
        _in_chroot("__debugpy_launcher.py"),
        *debugpy.get_args(debug_adapter),
    ]

    return RunDebugAdapterRequest(
        digest=merged_digest,
        args=args,
        extra_env=extra_env,
        append_only_caches=regular_run_request.append_only_caches,
        immutable_input_digests=regular_run_request.immutable_input_digests,
    )

1	# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	from __future__ import annotations	9✔
4
5	import dataclasses	9✔
6	import os	9✔
7	import textwrap	9✔
8	from collections.abc import Iterable	9✔
9
10	from pants.backend.python.subsystems.debugpy import DebugPy	9✔
11	from pants.backend.python.target_types import (	9✔
12	ConsoleScript,
13	Executable,
14	PexEntryPointField,
15	ResolvePexEntryPointRequest,
16	)
17	from pants.backend.python.target_types_rules import resolve_pex_entry_point	9✔
18	from pants.backend.python.util_rules.pex import Pex, VenvPexRequest, create_venv_pex	9✔
19	from pants.backend.python.util_rules.pex_environment import PexEnvironment	9✔
20	from pants.backend.python.util_rules.pex_from_targets import (	9✔
21	PexFromTargetsRequest,
22	create_pex_from_targets,
23	)
24	from pants.backend.python.util_rules.python_sources import (	9✔
25	PythonSourceFilesRequest,
26	prepare_python_sources,
27	)
28	from pants.core.goals.run import RunDebugAdapterRequest, RunRequest	9✔
29	from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem	9✔
30	from pants.engine.addresses import Address	9✔
31	from pants.engine.fs import CreateDigest, FileContent, MergeDigests	9✔
32	from pants.engine.internals.graph import transitive_targets as transitive_targets_get	9✔
33	from pants.engine.intrinsics import create_digest, merge_digests	9✔
34	from pants.engine.rules import concurrently, implicitly	9✔
35	from pants.engine.target import TransitiveTargetsRequest	9✔
36	from pants.util.frozendict import FrozenDict	9✔
37
38
39	def _in_chroot(relpath: str) -> str:	9✔
40	return os.path.join("{chroot}", relpath)	×
41
42
43	async def _create_python_source_run_request(	9✔
44	address: Address,
45	*,
46	entry_point_field: PexEntryPointField,
47	pex_env: PexEnvironment,
48	run_in_sandbox: bool,
49	pex_path: Iterable[Pex] = (),
50	console_script: ConsoleScript \| None = None,
51	executable: Executable \| None = None,
52	) -> RunRequest:
53	addresses = [address]	×
54	entry_point, transitive_targets = await concurrently(	×
55	resolve_pex_entry_point(ResolvePexEntryPointRequest(entry_point_field)),
56	transitive_targets_get(TransitiveTargetsRequest(addresses), **implicitly()),
57	)
58
59	pex_filename = (	×
60	address.generated_name.replace(".", "_") if address.generated_name else address.target_name
61	)
62
63	pex_request, sources = await concurrently(	×
64	create_pex_from_targets(
65	PexFromTargetsRequest(
66	addresses,
67	output_filename=f"{pex_filename}.pex",
68	internal_only=True,
69	include_source_files=False,
70	include_local_dists=True,
71	# `PEX_EXTRA_SYS_PATH` should contain this entry_point's module.
72	main=executable or console_script or entry_point.val,
73	additional_args=(
74	# N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH
75	# below, it's important for any app that re-executes itself that these environment
76	# variables are not stripped.
77	"--no-strip-pex-env",
78	),
79	),
80	**implicitly(),
81	),
82	prepare_python_sources(
83	PythonSourceFilesRequest(transitive_targets.closure, include_files=True), **implicitly()
84	),
85	)
86
87	pex_request = dataclasses.replace(pex_request, pex_path=(pex_request.pex_path, pex_path))	×
88
89	if run_in_sandbox:	×
90	# Note that a RunRequest always expects to run directly in the sandbox/workspace
91	# root, hence working_directory=None.
92	complete_pex_environment = pex_env.in_sandbox(working_directory=None)	×
93	else:
94	complete_pex_environment = pex_env.in_workspace()	×
NEW 95	venv_pex = await create_venv_pex(	×
96	VenvPexRequest(pex_request, complete_pex_environment), **implicitly()
97	)
98	input_digests = [	×
99	venv_pex.digest,
100	# Note regarding not-in-sandbox mode: You might think that the sources don't need to be copied
101	# into the chroot when using inline sources. But they do, because some of them might be
102	# codegenned, and those won't exist in the inline source tree. Rather than incurring the
103	# complexity of figuring out here which sources were codegenned, we copy everything.
104	# The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline
105	# sources will take precedence and their copies in the chroot will be ignored.
106	sources.source_files.snapshot.digest,
107	]
108	merged_digest = await merge_digests(MergeDigests(input_digests))	×
109
110	chrooted_source_roots = [_in_chroot(sr) for sr in sources.source_roots]	×
111	# The order here is important: we want the in-repo sources to take precedence over their
112	# copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
113	source_roots = [	×
114	*([] if run_in_sandbox else sources.source_roots),
115	*chrooted_source_roots,
116	]
117	extra_env = {	×
118	**complete_pex_environment.environment_dict(python_configured=True),
119	"PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots),
120	}
121	append_only_caches = (	×
122	FrozenDict({}) if venv_pex.append_only_caches is None else venv_pex.append_only_caches
123	)
124
125	return RunRequest(	×
126	digest=merged_digest,
127	args=[_in_chroot(venv_pex.pex.argv0)],
128	extra_env=extra_env,
129	append_only_caches={
130	**complete_pex_environment.append_only_caches,
131	**append_only_caches,
132	},
133	)
134
135
136	async def _create_python_source_run_dap_request(	9✔
137	regular_run_request: RunRequest,
138	*,
139	debugpy: DebugPy,
140	debug_adapter: DebugAdapterSubsystem,
141	run_in_sandbox: bool,
142	) -> RunDebugAdapterRequest:
143	launcher_digest = await create_digest(	×
144	CreateDigest(
145	[
146	FileContent(
147	"__debugpy_launcher.py",
148	textwrap.dedent(
149	"""
150	import os
151	CHROOT = os.environ["PANTS_CHROOT"]
152
153	del os.environ["PEX_INTERPRETER"]
154
155	import debugpy._vendored.force_pydevd
156	from _pydevd_bundle.pydevd_process_net_command_json import PyDevJsonCommandProcessor
157	orig_resolve_remote_root = PyDevJsonCommandProcessor._resolve_remote_root
158
159	def patched_resolve_remote_root(self, local_root, remote_root):
160	if remote_root == ".":
161	remote_root = CHROOT
162	return orig_resolve_remote_root(self, local_root, remote_root)
163
164	PyDevJsonCommandProcessor._resolve_remote_root = patched_resolve_remote_root
165
166	from debugpy.server import cli
167	cli.main()
168	"""
169	).encode("utf-8"),
170	),
171	]
172	),
173	)
174
175	merged_digest = await merge_digests(MergeDigests([regular_run_request.digest, launcher_digest]))	×
176	extra_env = dict(regular_run_request.extra_env)	×
177	extra_env["PEX_INTERPRETER"] = "1"	×
178	# See https://github.com/pantsbuild/pants/issues/17540
179	# and https://github.com/pantsbuild/pants/issues/18243
180	# For `run --debug-adapter`, the client might send a `pathMappings`
181	# (this is likely as VS Code likes to configure that by default) with a `remoteRoot` of ".".
182	#
183	# For `run`, CWD is the build root. If `run_in_sandbox` is False, everything is OK.
184	# If `run_in_sandbox` is True, breakpoints won't be hit as CWD != sandbox root.
185	#
186	# We fix this by monkeypatching pydevd (the library powering debugpy) so that a remoteRoot of "."
187	# means the sandbox root.
188	# See https://github.com/fabioz/PyDev.Debugger/pull/243 for a better solution.
189	extra_env["PANTS_CHROOT"] = _in_chroot("").rstrip("/") if run_in_sandbox else "."	×
190	args = [	×
191	*regular_run_request.args,
192	_in_chroot("__debugpy_launcher.py"),
193	*debugpy.get_args(debug_adapter),
194	]
195
196	return RunDebugAdapterRequest(	×
197	digest=merged_digest,
198	args=args,
199	extra_env=extra_env,
200	append_only_caches=regular_run_request.append_only_caches,
201	immutable_input_digests=regular_run_request.immutable_input_digests,
202	)

pantsbuild / pants / 20937871815

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous