24145945949

Committed 08 Apr 2026 04:14PM UTC coverage: 82.077% (-10.8%) from 92.91%

Build # 24145945949

Build Type

Pull #23233

github

Committed by

web-flow

Commit Message

Merge 089d98e3c into 9036734c9

Pull Request Pull Request #23233: Introduce a LockfileFormat enum.

Coverage Stats

8 of 11 new or added lines in 4 files covered. (72.73%)

7635 existing lines in 306 files now uncovered.

63732 of 77649 relevant lines covered (82.08%)

2.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.7

/src/python/pants/backend/python/goals/lockfile.py

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import itertools
import os.path
from collections import defaultdict
from dataclasses import dataclass
from operator import itemgetter

from pants.backend.python.subsystems.python_tool_base import PythonToolBase
from pants.backend.python.subsystems.setup import PythonSetup
from pants.backend.python.target_types import (
    PythonRequirementFindLinksField,
    PythonRequirementResolveField,
    PythonRequirementsField,
)
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.lockfile_diff import _generate_python_lockfile_diff
from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadata
from pants.backend.python.util_rules.pex import (
    CompletePlatforms,
    digest_complete_platform_addresses,
    find_interpreter,
)
from pants.backend.python.util_rules.pex_cli import PexCliProcess, maybe_log_pex_stderr
from pants.backend.python.util_rules.pex_environment import PexSubsystem
from pants.backend.python.util_rules.pex_requirements import (
    PexRequirements,
    ResolvePexConfig,
    ResolvePexConfigRequest,
    determine_resolve_pex_config,
)
from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
from pants.core.goals.generate_lockfiles import (
    DEFAULT_TOOL_LOCKFILE,
    GenerateLockfile,
    GenerateLockfileResult,
    GenerateLockfilesSubsystem,
    KnownUserResolveNames,
    KnownUserResolveNamesRequest,
    RequestedUserResolveNames,
    UserGenerateLockfiles,
    WrappedGenerateLockfile,
)
from pants.core.goals.resolves import ExportableTool
from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest
from pants.engine.addresses import UnparsedAddressInputs
from pants.engine.fs import (
    CreateDigest,
    Digest,
    FileContent,
    GlobExpansionConjunction,
    MergeDigests,
    PathGlobs,
)
from pants.engine.internals.native_engine import EMPTY_DIGEST
from pants.engine.internals.synthetic_targets import SyntheticAddressMaps, SyntheticTargetsRequest
from pants.engine.internals.target_adaptor import TargetAdaptor
from pants.engine.intrinsics import (
    create_digest,
    get_digest_contents,
    merge_digests,
    path_globs_to_digest,
)
from pants.engine.process import ProcessCacheScope, execute_process_or_raise
from pants.engine.rules import collect_rules, implicitly, rule
from pants.engine.target import AllTargets
from pants.engine.unions import UnionMembership, UnionRule
from pants.option.subsystem import _construct_subsystem
from pants.util.docutil import bin_name
from pants.util.logging import LogLevel
from pants.util.ordered_set import FrozenOrderedSet
from pants.util.pip_requirement import PipRequirement


@dataclass(frozen=True)
class GeneratePythonLockfile(GenerateLockfile):
    requirements: FrozenOrderedSet[str]
    find_links: FrozenOrderedSet[str]
    interpreter_constraints: InterpreterConstraints
    lock_style: str
    complete_platforms: tuple[str, ...]

    @property
    def requirements_hex_digest(self) -> str:
        """Produces a hex digest of the requirements input for this lockfile."""
        return calculate_invalidation_digest(self.requirements)


@rule
async def wrap_python_lockfile_request(request: GeneratePythonLockfile) -> WrappedGenerateLockfile:
    return WrappedGenerateLockfile(request)


@dataclass(frozen=True)
class _PipArgsAndConstraintsSetup:
    resolve_config: ResolvePexConfig
    args: tuple[str, ...]
    digest: Digest


async def _setup_pip_args_and_constraints_file(resolve_name: str) -> _PipArgsAndConstraintsSetup:
    resolve_config = await determine_resolve_pex_config(
        ResolvePexConfigRequest(resolve_name), **implicitly()
    )

    args = list(resolve_config.pex_args())
    digests: list[Digest] = []

    if resolve_config.constraints_file:
        args.append(f"--constraints={resolve_config.constraints_file.path}")
        digests.append(resolve_config.constraints_file.digest)

    input_digest = await merge_digests(MergeDigests(digests))
    return _PipArgsAndConstraintsSetup(resolve_config, tuple(args), input_digest)


@rule(desc="Generate Python lockfile", level=LogLevel.DEBUG)
async def generate_lockfile(
    req: GeneratePythonLockfile,
    generate_lockfiles_subsystem: GenerateLockfilesSubsystem,
    python_setup: PythonSetup,
    pex_subsystem: PexSubsystem,
) -> GenerateLockfileResult:
    if not req.requirements:
        raise ValueError(
            f"Cannot generate lockfile with no requirements. Please add some requirements to {req.resolve_name}."
        )

    pip_args_setup = await _setup_pip_args_and_constraints_file(req.resolve_name)
    header_delimiter = "//"

    python = await find_interpreter(req.interpreter_constraints, **implicitly())

    # Resolve complete platform targets if specified
    complete_platforms: CompletePlatforms | None = None
    if req.complete_platforms:
        # Resolve target addresses to get platform JSON files
        complete_platforms = await digest_complete_platform_addresses(
            UnparsedAddressInputs(
                req.complete_platforms,
                owning_address=None,
                description_of_origin=f"the `[python].resolves_to_complete_platforms` for resolve `{req.resolve_name}`",
            )
        )

    # Add complete platforms if specified, otherwise use default target systems for universal locks
    if complete_platforms:
        target_system_args = tuple(
            f"--complete-platform={platform}" for platform in complete_platforms
        )
    elif req.lock_style == "universal":
        # PEX files currently only run on Linux and Mac machines; so we hard code this
        # limit on lock universality to avoid issues locking due to irrelevant
        # Windows-only dependency issues. See this Pex issue that originated from a
        # Pants user issue presented in Slack:
        #   https://github.com/pex-tool/pex/issues/1821
        #
        # Note: --target-system only applies to universal locks. For other lock styles
        # (strict, sources) without complete platforms, we don't specify platform args
        # and PEX will lock for the current platform only.
        target_system_args = (
            "--target-system",
            "linux",
            "--target-system",
            "mac",
        )
    else:
        # For non-universal lock styles without complete platforms, don't specify
        # platform arguments - PEX will lock for the current platform only
        target_system_args = ()

    if generate_lockfiles_subsystem.sync:
        existing_lockfile_digest = await path_globs_to_digest(
            PathGlobs(
                globs=(req.lockfile_dest,),
                # We ignore errors, since the lockfile may not exist.
                glob_match_error_behavior=GlobMatchErrorBehavior.ignore,
                conjunction=GlobExpansionConjunction.any_match,
            )
        )
    else:
        existing_lockfile_digest = EMPTY_DIGEST

    output_flag = "--lock" if generate_lockfiles_subsystem.sync else "--output"
    result = await execute_process_or_raise(
        **implicitly(
            PexCliProcess(
                subcommand=("lock", "sync" if generate_lockfiles_subsystem.sync else "create"),
                extra_args=(
                    f"{output_flag}={req.lockfile_dest}",
                    f"--style={req.lock_style}",
                    "--pip-version",
                    python_setup.pip_version,
                    "--resolver-version",
                    "pip-2020-resolver",
                    "--preserve-pip-download-log",
                    "pex-pip-download.log",
                    *target_system_args,
                    # This makes diffs more readable when lockfiles change.
                    "--indent=2",
                    f"--python-path={python.path}",
                    *(f"--find-links={link}" for link in req.find_links),
                    *pip_args_setup.args,
                    # When complete platforms are specified, don't pass interpreter constraints.
                    # The complete platforms already define Python versions and platforms.
                    # Passing both causes PEX to generate duplicate locked_requirements entries
                    # when the local platform matches a complete platform.
                    # TODO(#9560): Consider validating that these platforms are valid with the
                    #  interpreter constraints.
                    *(
                        req.interpreter_constraints.generate_pex_arg_list()
                        if not complete_platforms
                        else ()
                    ),
                    *(
                        f"--override={override}"
                        for override in pip_args_setup.resolve_config.overrides
                    ),
                    *req.requirements,
                ),
                additional_input_digest=await merge_digests(
                    MergeDigests(
                        [existing_lockfile_digest, pip_args_setup.digest]
                        + ([complete_platforms.digest] if complete_platforms else [])
                    )
                ),
                output_files=(req.lockfile_dest,),
                description=f"Generate lockfile for {req.resolve_name}",
                # Instead of caching lockfile generation with LMDB, we instead use the invalidation
                # scheme from `lockfile_metadata.py` to check for stale/invalid lockfiles. This is
                # necessary so that our invalidation is resilient to deleting LMDB or running on a
                # new machine.
                #
                # We disable caching with LMDB so that when you generate a lockfile, you always get
                # the most up-to-date snapshot of the world. This is generally desirable and also
                # necessary to avoid an awkward edge case where different developers generate
                # different lockfiles even when generating at the same time. See
                # https://github.com/pantsbuild/pants/issues/12591.
                cache_scope=ProcessCacheScope.PER_SESSION,
            )
        )
    )
    maybe_log_pex_stderr(result.stderr, pex_subsystem.verbosity)

    metadata = PythonLockfileMetadata.new(
        valid_for_interpreter_constraints=req.interpreter_constraints,
        requirements={
            PipRequirement.parse(
                i,
                description_of_origin=f"the lockfile {req.lockfile_dest} for the resolve {req.resolve_name}",
            )
            for i in req.requirements
        },
        manylinux=pip_args_setup.resolve_config.manylinux,
        requirement_constraints=(
            set(pip_args_setup.resolve_config.constraints_file.constraints)
            if pip_args_setup.resolve_config.constraints_file
            else set()
        ),
        only_binary=set(pip_args_setup.resolve_config.only_binary),
        no_binary=set(pip_args_setup.resolve_config.no_binary),
        excludes=set(pip_args_setup.resolve_config.excludes),
        overrides=set(pip_args_setup.resolve_config.overrides),
        sources=set(pip_args_setup.resolve_config.sources),
        lock_style=req.lock_style,
        complete_platforms=req.complete_platforms,
        uploaded_prior_to=pip_args_setup.resolve_config.uploaded_prior_to,
    )
    regenerate_command = (
        generate_lockfiles_subsystem.custom_command
        or f"{bin_name()} generate-lockfiles --resolve={req.resolve_name}"
    )
    if python_setup.separate_lockfile_metadata_file:
        descr = f"This lockfile was generated by Pants. To regenerate, run: {regenerate_command}"
        metadata_digest = await create_digest(
            CreateDigest(
                [
                    FileContent(
                        PythonLockfileMetadata.metadata_location_for_lockfile(req.lockfile_dest),
                        metadata.to_json(with_description=descr).encode(),
                    ),
                ]
            )
        )
        final_lockfile_digest = await merge_digests(
            MergeDigests([metadata_digest, result.output_digest])
        )
    else:
        initial_lockfile_digest_contents = await get_digest_contents(result.output_digest)
        lockfile_with_header = metadata.add_header_to_lockfile(
            initial_lockfile_digest_contents[0].content,
            regenerate_command=regenerate_command,
            delimeter=header_delimiter,
        )
        final_lockfile_digest = await create_digest(
            CreateDigest(
                [
                    FileContent(req.lockfile_dest, lockfile_with_header),
                ]
            )
        )

    if req.diff:
        diff = await _generate_python_lockfile_diff(
            final_lockfile_digest, req.resolve_name, req.lockfile_dest
        )
    else:
        diff = None

    return GenerateLockfileResult(final_lockfile_digest, req.resolve_name, req.lockfile_dest, diff)


class RequestedPythonUserResolveNames(RequestedUserResolveNames):
    pass


class KnownPythonUserResolveNamesRequest(KnownUserResolveNamesRequest):
    pass


@rule
async def determine_python_user_resolves(
    _: KnownPythonUserResolveNamesRequest,
    python_setup: PythonSetup,
    union_membership: UnionMembership,
) -> KnownUserResolveNames:
    """Find all know Python resolves, from both user-created resolves and internal tools."""
    python_tool_resolves = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase)

    tools_using_default_resolve = [
        resolve_name
        for resolve_name, subsystem_cls in python_tool_resolves.items()
        if (await _construct_subsystem(subsystem_cls)).install_from_resolve is None
    ]

    return KnownUserResolveNames(
        names=(
            *python_setup.resolves.keys(),
            *tools_using_default_resolve,
        ),  # the order of the keys doesn't matter since shadowing is done in `setup_user_lockfile_requests`
        option_name="[python].resolves",
        requested_resolve_names_cls=RequestedPythonUserResolveNames,
    )


@rule
async def setup_user_lockfile_requests(
    requested: RequestedPythonUserResolveNames,
    all_targets: AllTargets,
    python_setup: PythonSetup,
    union_membership: UnionMembership,
) -> UserGenerateLockfiles:
    """Transform the names of resolves requested into the `GeneratePythonLockfile` request object.

    Shadowing is done here by only checking internal resolves if the resolve is not a user-created
    resolve.
    """
    if not (python_setup.enable_resolves and python_setup.resolves_generate_lockfiles):
        return UserGenerateLockfiles()

    resolve_to_requirements_fields = defaultdict(set)
    resolve_to_find_links: dict[str, set[str]] = defaultdict(set)
    for tgt in all_targets:
        if not tgt.has_fields((PythonRequirementResolveField, PythonRequirementsField)):
            continue
        resolve = tgt[PythonRequirementResolveField].normalized_value(python_setup)
        resolve_to_requirements_fields[resolve].add(tgt[PythonRequirementsField])
        resolve_to_find_links[resolve].update(tgt[PythonRequirementFindLinksField].value or ())

    tools = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase)

    out = set()
    for resolve in requested:
        if resolve in python_setup.resolves:
            out.add(
                GeneratePythonLockfile(
                    requirements=PexRequirements.req_strings_from_requirement_fields(
                        resolve_to_requirements_fields[resolve]
                    ),
                    find_links=FrozenOrderedSet(resolve_to_find_links[resolve]),
                    interpreter_constraints=InterpreterConstraints(
                        python_setup.resolves_to_interpreter_constraints.get(
                            resolve, python_setup.interpreter_constraints
                        )
                    ),
                    resolve_name=resolve,
                    lockfile_dest=python_setup.resolves[resolve],
                    diff=False,
                    lock_style=python_setup.resolves_to_lock_style().get(resolve, "universal"),
                    complete_platforms=tuple(
                        python_setup.resolves_to_complete_platforms().get(resolve, [])
                    ),
                )
            )
        else:
            tool_cls: type[PythonToolBase] = tools[resolve]
            tool = await _construct_subsystem(tool_cls)

            # TODO: we shouldn't be managing default ICs in lockfile identification.
            #   We should find a better place to do this or a better way to default
            if tool.register_interpreter_constraints:
                ic = tool.interpreter_constraints
            else:
                ic = InterpreterConstraints(tool.default_interpreter_constraints)

            out.add(
                GeneratePythonLockfile(
                    requirements=FrozenOrderedSet(sorted(tool.requirements)),
                    find_links=FrozenOrderedSet(),
                    interpreter_constraints=ic,
                    resolve_name=resolve,
                    lockfile_dest=DEFAULT_TOOL_LOCKFILE,
                    diff=False,
                    lock_style="universal",  # Tools always use universal style
                    complete_platforms=(),  # Tools don't use complete platforms
                )
            )

    return UserGenerateLockfiles(out)


@dataclass(frozen=True)
class PythonSyntheticLockfileTargetsRequest(SyntheticTargetsRequest):
    """Register the type used to create synthetic targets for Python lockfiles.

    As the paths for all lockfiles are known up-front, we set the `path` field to
    `SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS` so that we get a single request for all
    our synthetic targets rather than one request per directory.
    """

    path: str = SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS


def synthetic_lockfile_target_name(resolve: str) -> str:
    return f"_{resolve}_lockfile"


@rule
async def python_lockfile_synthetic_targets(
    request: PythonSyntheticLockfileTargetsRequest,
    python_setup: PythonSetup,
) -> SyntheticAddressMaps:
    if not python_setup.enable_synthetic_lockfiles:
        return SyntheticAddressMaps()

    resolves = [
        (os.path.dirname(lockfile), os.path.basename(lockfile), name)
        for name, lockfile in python_setup.resolves.items()
    ]

    return SyntheticAddressMaps.for_targets_request(
        request,
        [
            (
                os.path.join(spec_path, "BUILD.python-lockfiles"),
                tuple(
                    TargetAdaptor(
                        "_lockfiles",
                        name=synthetic_lockfile_target_name(name),
                        sources=(lockfile,),
                        __description_of_origin__=f"the [python].resolves option {name!r}",
                    )
                    for _, lockfile, name in lockfiles
                ),
            )
            for spec_path, lockfiles in itertools.groupby(sorted(resolves), key=itemgetter(0))
        ],
    )


def rules():
    return (
        *collect_rules(),
        UnionRule(GenerateLockfile, GeneratePythonLockfile),
        UnionRule(KnownUserResolveNamesRequest, KnownPythonUserResolveNamesRequest),
        UnionRule(RequestedUserResolveNames, RequestedPythonUserResolveNames),
        UnionRule(SyntheticTargetsRequest, PythonSyntheticLockfileTargetsRequest),
    )

1	# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	7✔
5
6	import itertools	7✔
7	import os.path	7✔
8	from collections import defaultdict	7✔
9	from dataclasses import dataclass	7✔
10	from operator import itemgetter	7✔
11
12	from pants.backend.python.subsystems.python_tool_base import PythonToolBase	7✔
13	from pants.backend.python.subsystems.setup import PythonSetup	7✔
14	from pants.backend.python.target_types import (	7✔
15	PythonRequirementFindLinksField,
16	PythonRequirementResolveField,
17	PythonRequirementsField,
18	)
19	from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints	7✔
20	from pants.backend.python.util_rules.lockfile_diff import _generate_python_lockfile_diff	7✔
21	from pants.backend.python.util_rules.lockfile_metadata import PythonLockfileMetadata	7✔
22	from pants.backend.python.util_rules.pex import (	7✔
23	CompletePlatforms,
24	digest_complete_platform_addresses,
25	find_interpreter,
26	)
27	from pants.backend.python.util_rules.pex_cli import PexCliProcess, maybe_log_pex_stderr	7✔
28	from pants.backend.python.util_rules.pex_environment import PexSubsystem	7✔
29	from pants.backend.python.util_rules.pex_requirements import (	7✔
30	PexRequirements,
31	ResolvePexConfig,
32	ResolvePexConfigRequest,
33	determine_resolve_pex_config,
34	)
35	from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior	7✔
36	from pants.core.goals.generate_lockfiles import (	7✔
37	DEFAULT_TOOL_LOCKFILE,
38	GenerateLockfile,
39	GenerateLockfileResult,
40	GenerateLockfilesSubsystem,
41	KnownUserResolveNames,
42	KnownUserResolveNamesRequest,
43	RequestedUserResolveNames,
44	UserGenerateLockfiles,
45	WrappedGenerateLockfile,
46	)
47	from pants.core.goals.resolves import ExportableTool	7✔
48	from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest	7✔
49	from pants.engine.addresses import UnparsedAddressInputs	7✔
50	from pants.engine.fs import (	7✔
51	CreateDigest,
52	Digest,
53	FileContent,
54	GlobExpansionConjunction,
55	MergeDigests,
56	PathGlobs,
57	)
58	from pants.engine.internals.native_engine import EMPTY_DIGEST	7✔
59	from pants.engine.internals.synthetic_targets import SyntheticAddressMaps, SyntheticTargetsRequest	7✔
60	from pants.engine.internals.target_adaptor import TargetAdaptor	7✔
61	from pants.engine.intrinsics import (	7✔
62	create_digest,
63	get_digest_contents,
64	merge_digests,
65	path_globs_to_digest,
66	)
67	from pants.engine.process import ProcessCacheScope, execute_process_or_raise	7✔
68	from pants.engine.rules import collect_rules, implicitly, rule	7✔
69	from pants.engine.target import AllTargets	7✔
70	from pants.engine.unions import UnionMembership, UnionRule	7✔
71	from pants.option.subsystem import _construct_subsystem	7✔
72	from pants.util.docutil import bin_name	7✔
73	from pants.util.logging import LogLevel	7✔
74	from pants.util.ordered_set import FrozenOrderedSet	7✔
75	from pants.util.pip_requirement import PipRequirement	7✔
76
77
78	@dataclass(frozen=True)	7✔
79	class GeneratePythonLockfile(GenerateLockfile):	7✔
80	requirements: FrozenOrderedSet[str]	7✔
81	find_links: FrozenOrderedSet[str]	7✔
82	interpreter_constraints: InterpreterConstraints	7✔
83	lock_style: str	7✔
84	complete_platforms: tuple[str, ...]	7✔
85
86	@property	7✔
87	def requirements_hex_digest(self) -> str:	7✔
88	"""Produces a hex digest of the requirements input for this lockfile."""
89	return calculate_invalidation_digest(self.requirements)	×
90
91
92	@rule	7✔
93	async def wrap_python_lockfile_request(request: GeneratePythonLockfile) -> WrappedGenerateLockfile:	7✔
94	return WrappedGenerateLockfile(request)	×
95
96
97	@dataclass(frozen=True)	7✔
98	class _PipArgsAndConstraintsSetup:	7✔
99	resolve_config: ResolvePexConfig	7✔
100	args: tuple[str, ...]	7✔
101	digest: Digest	7✔
102
103
104	async def _setup_pip_args_and_constraints_file(resolve_name: str) -> _PipArgsAndConstraintsSetup:	7✔
UNCOV 105	resolve_config = await determine_resolve_pex_config(	×
106	ResolvePexConfigRequest(resolve_name), **implicitly()
107	)
108
UNCOV 109	args = list(resolve_config.pex_args())	×
UNCOV 110	digests: list[Digest] = []	×
111
UNCOV 112	if resolve_config.constraints_file:	×
UNCOV 113	args.append(f"--constraints={resolve_config.constraints_file.path}")	×
UNCOV 114	digests.append(resolve_config.constraints_file.digest)	×
115
UNCOV 116	input_digest = await merge_digests(MergeDigests(digests))	×
UNCOV 117	return _PipArgsAndConstraintsSetup(resolve_config, tuple(args), input_digest)	×
118
119
120	@rule(desc="Generate Python lockfile", level=LogLevel.DEBUG)	7✔
121	async def generate_lockfile(	7✔
122	req: GeneratePythonLockfile,
123	generate_lockfiles_subsystem: GenerateLockfilesSubsystem,
124	python_setup: PythonSetup,
125	pex_subsystem: PexSubsystem,
126	) -> GenerateLockfileResult:
UNCOV 127	if not req.requirements:	×
UNCOV 128	raise ValueError(	×
129	f"Cannot generate lockfile with no requirements. Please add some requirements to {req.resolve_name}."
130	)
131
UNCOV 132	pip_args_setup = await _setup_pip_args_and_constraints_file(req.resolve_name)	×
UNCOV 133	header_delimiter = "//"	×
134
UNCOV 135	python = await find_interpreter(req.interpreter_constraints, **implicitly())	×
136
137	# Resolve complete platform targets if specified
UNCOV 138	complete_platforms: CompletePlatforms \| None = None	×
UNCOV 139	if req.complete_platforms:	×
140	# Resolve target addresses to get platform JSON files
141	complete_platforms = await digest_complete_platform_addresses(	×
142	UnparsedAddressInputs(
143	req.complete_platforms,
144	owning_address=None,
145	description_of_origin=f"the `[python].resolves_to_complete_platforms` for resolve `{req.resolve_name}`",
146	)
147	)
148
149	# Add complete platforms if specified, otherwise use default target systems for universal locks
UNCOV 150	if complete_platforms:	×
151	target_system_args = tuple(	×
152	f"--complete-platform={platform}" for platform in complete_platforms
153	)
UNCOV 154	elif req.lock_style == "universal":	×
155	# PEX files currently only run on Linux and Mac machines; so we hard code this
156	# limit on lock universality to avoid issues locking due to irrelevant
157	# Windows-only dependency issues. See this Pex issue that originated from a
158	# Pants user issue presented in Slack:
159	# https://github.com/pex-tool/pex/issues/1821
160	#
161	# Note: --target-system only applies to universal locks. For other lock styles
162	# (strict, sources) without complete platforms, we don't specify platform args
163	# and PEX will lock for the current platform only.
UNCOV 164	target_system_args = (	×
165	"--target-system",
166	"linux",
167	"--target-system",
168	"mac",
169	)
170	else:
171	# For non-universal lock styles without complete platforms, don't specify
172	# platform arguments - PEX will lock for the current platform only
173	target_system_args = ()	×
174
UNCOV 175	if generate_lockfiles_subsystem.sync:	×
176	existing_lockfile_digest = await path_globs_to_digest(	×
177	PathGlobs(
178	globs=(req.lockfile_dest,),
179	# We ignore errors, since the lockfile may not exist.
180	glob_match_error_behavior=GlobMatchErrorBehavior.ignore,
181	conjunction=GlobExpansionConjunction.any_match,
182	)
183	)
184	else:
UNCOV 185	existing_lockfile_digest = EMPTY_DIGEST	×
186
UNCOV 187	output_flag = "--lock" if generate_lockfiles_subsystem.sync else "--output"	×
UNCOV 188	result = await execute_process_or_raise(	×
189	**implicitly(
190	PexCliProcess(
191	subcommand=("lock", "sync" if generate_lockfiles_subsystem.sync else "create"),
192	extra_args=(
193	f"{output_flag}={req.lockfile_dest}",
194	f"--style={req.lock_style}",
195	"--pip-version",
196	python_setup.pip_version,
197	"--resolver-version",
198	"pip-2020-resolver",
199	"--preserve-pip-download-log",
200	"pex-pip-download.log",
201	*target_system_args,
202	# This makes diffs more readable when lockfiles change.
203	"--indent=2",
204	f"--python-path={python.path}",
205	*(f"--find-links={link}" for link in req.find_links),
206	*pip_args_setup.args,
207	# When complete platforms are specified, don't pass interpreter constraints.
208	# The complete platforms already define Python versions and platforms.
209	# Passing both causes PEX to generate duplicate locked_requirements entries
210	# when the local platform matches a complete platform.
211	# TODO(#9560): Consider validating that these platforms are valid with the
212	# interpreter constraints.
213	*(
214	req.interpreter_constraints.generate_pex_arg_list()
215	if not complete_platforms
216	else ()
217	),
218	*(
219	f"--override={override}"
220	for override in pip_args_setup.resolve_config.overrides
221	),
222	*req.requirements,
223	),
224	additional_input_digest=await merge_digests(
225	MergeDigests(
226	[existing_lockfile_digest, pip_args_setup.digest]
227	+ ([complete_platforms.digest] if complete_platforms else [])
228	)
229	),
230	output_files=(req.lockfile_dest,),
231	description=f"Generate lockfile for {req.resolve_name}",
232	# Instead of caching lockfile generation with LMDB, we instead use the invalidation
233	# scheme from `lockfile_metadata.py` to check for stale/invalid lockfiles. This is
234	# necessary so that our invalidation is resilient to deleting LMDB or running on a
235	# new machine.
236	#
237	# We disable caching with LMDB so that when you generate a lockfile, you always get
238	# the most up-to-date snapshot of the world. This is generally desirable and also
239	# necessary to avoid an awkward edge case where different developers generate
240	# different lockfiles even when generating at the same time. See
241	# https://github.com/pantsbuild/pants/issues/12591.
242	cache_scope=ProcessCacheScope.PER_SESSION,
243	)
244	)
245	)
UNCOV 246	maybe_log_pex_stderr(result.stderr, pex_subsystem.verbosity)	×
247
UNCOV 248	metadata = PythonLockfileMetadata.new(	×
249	valid_for_interpreter_constraints=req.interpreter_constraints,
250	requirements={
251	PipRequirement.parse(
252	i,
253	description_of_origin=f"the lockfile {req.lockfile_dest} for the resolve {req.resolve_name}",
254	)
255	for i in req.requirements
256	},
257	manylinux=pip_args_setup.resolve_config.manylinux,
258	requirement_constraints=(
259	set(pip_args_setup.resolve_config.constraints_file.constraints)
260	if pip_args_setup.resolve_config.constraints_file
261	else set()
262	),
263	only_binary=set(pip_args_setup.resolve_config.only_binary),
264	no_binary=set(pip_args_setup.resolve_config.no_binary),
265	excludes=set(pip_args_setup.resolve_config.excludes),
266	overrides=set(pip_args_setup.resolve_config.overrides),
267	sources=set(pip_args_setup.resolve_config.sources),
268	lock_style=req.lock_style,
269	complete_platforms=req.complete_platforms,
270	uploaded_prior_to=pip_args_setup.resolve_config.uploaded_prior_to,
271	)
UNCOV 272	regenerate_command = (	×
273	generate_lockfiles_subsystem.custom_command
274	or f"{bin_name()} generate-lockfiles --resolve={req.resolve_name}"
275	)
UNCOV 276	if python_setup.separate_lockfile_metadata_file:	×
UNCOV 277	descr = f"This lockfile was generated by Pants. To regenerate, run: {regenerate_command}"	×
UNCOV 278	metadata_digest = await create_digest(	×
279	CreateDigest(
280	[
281	FileContent(
282	PythonLockfileMetadata.metadata_location_for_lockfile(req.lockfile_dest),
283	metadata.to_json(with_description=descr).encode(),
284	),
285	]
286	)
287	)
UNCOV 288	final_lockfile_digest = await merge_digests(	×
289	MergeDigests([metadata_digest, result.output_digest])
290	)
291	else:
UNCOV 292	initial_lockfile_digest_contents = await get_digest_contents(result.output_digest)	×
UNCOV 293	lockfile_with_header = metadata.add_header_to_lockfile(	×
294	initial_lockfile_digest_contents[0].content,
295	regenerate_command=regenerate_command,
296	delimeter=header_delimiter,
297	)
UNCOV 298	final_lockfile_digest = await create_digest(	×
299	CreateDigest(
300	[
301	FileContent(req.lockfile_dest, lockfile_with_header),
302	]
303	)
304	)
305
UNCOV 306	if req.diff:	×
307	diff = await _generate_python_lockfile_diff(	×
308	final_lockfile_digest, req.resolve_name, req.lockfile_dest
309	)
310	else:
UNCOV 311	diff = None	×
312
UNCOV 313	return GenerateLockfileResult(final_lockfile_digest, req.resolve_name, req.lockfile_dest, diff)	×
314
315
316	class RequestedPythonUserResolveNames(RequestedUserResolveNames):	7✔
317	pass	7✔
318
319
320	class KnownPythonUserResolveNamesRequest(KnownUserResolveNamesRequest):	7✔
321	pass	7✔
322
323
324	@rule	7✔
325	async def determine_python_user_resolves(	7✔
326	_: KnownPythonUserResolveNamesRequest,
327	python_setup: PythonSetup,
328	union_membership: UnionMembership,
329	) -> KnownUserResolveNames:
330	"""Find all know Python resolves, from both user-created resolves and internal tools."""
UNCOV 331	python_tool_resolves = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase)	×
332
UNCOV 333	tools_using_default_resolve = [	×
334	resolve_name
335	for resolve_name, subsystem_cls in python_tool_resolves.items()
336	if (await _construct_subsystem(subsystem_cls)).install_from_resolve is None
337	]
338
UNCOV 339	return KnownUserResolveNames(	×
340	names=(
341	*python_setup.resolves.keys(),
342	*tools_using_default_resolve,
343	), # the order of the keys doesn't matter since shadowing is done in `setup_user_lockfile_requests`
344	option_name="[python].resolves",
345	requested_resolve_names_cls=RequestedPythonUserResolveNames,
346	)
347
348
349	@rule	7✔
350	async def setup_user_lockfile_requests(	7✔
351	requested: RequestedPythonUserResolveNames,
352	all_targets: AllTargets,
353	python_setup: PythonSetup,
354	union_membership: UnionMembership,
355	) -> UserGenerateLockfiles:
356	"""Transform the names of resolves requested into the `GeneratePythonLockfile` request object.
357
358	Shadowing is done here by only checking internal resolves if the resolve is not a user-created
359	resolve.
360	"""
UNCOV 361	if not (python_setup.enable_resolves and python_setup.resolves_generate_lockfiles):	×
362	return UserGenerateLockfiles()	×
363
UNCOV 364	resolve_to_requirements_fields = defaultdict(set)	×
UNCOV 365	resolve_to_find_links: dict[str, set[str]] = defaultdict(set)	×
UNCOV 366	for tgt in all_targets:	×
UNCOV 367	if not tgt.has_fields((PythonRequirementResolveField, PythonRequirementsField)):	×
368	continue	×
UNCOV 369	resolve = tgt[PythonRequirementResolveField].normalized_value(python_setup)	×
UNCOV 370	resolve_to_requirements_fields[resolve].add(tgt[PythonRequirementsField])	×
UNCOV 371	resolve_to_find_links[resolve].update(tgt[PythonRequirementFindLinksField].value or ())	×
372
UNCOV 373	tools = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase)	×
374
UNCOV 375	out = set()	×
UNCOV 376	for resolve in requested:	×
UNCOV 377	if resolve in python_setup.resolves:	×
UNCOV 378	out.add(	×
379	GeneratePythonLockfile(
380	requirements=PexRequirements.req_strings_from_requirement_fields(
381	resolve_to_requirements_fields[resolve]
382	),
383	find_links=FrozenOrderedSet(resolve_to_find_links[resolve]),
384	interpreter_constraints=InterpreterConstraints(
385	python_setup.resolves_to_interpreter_constraints.get(
386	resolve, python_setup.interpreter_constraints
387	)
388	),
389	resolve_name=resolve,
390	lockfile_dest=python_setup.resolves[resolve],
391	diff=False,
392	lock_style=python_setup.resolves_to_lock_style().get(resolve, "universal"),
393	complete_platforms=tuple(
394	python_setup.resolves_to_complete_platforms().get(resolve, [])
395	),
396	)
397	)
398	else:
399	tool_cls: type[PythonToolBase] = tools[resolve]	×
400	tool = await _construct_subsystem(tool_cls)	×
401
402	# TODO: we shouldn't be managing default ICs in lockfile identification.
403	# We should find a better place to do this or a better way to default
404	if tool.register_interpreter_constraints:	×
405	ic = tool.interpreter_constraints	×
406	else:
407	ic = InterpreterConstraints(tool.default_interpreter_constraints)	×
408
409	out.add(	×
410	GeneratePythonLockfile(
411	requirements=FrozenOrderedSet(sorted(tool.requirements)),
412	find_links=FrozenOrderedSet(),
413	interpreter_constraints=ic,
414	resolve_name=resolve,
415	lockfile_dest=DEFAULT_TOOL_LOCKFILE,
416	diff=False,
417	lock_style="universal", # Tools always use universal style
418	complete_platforms=(), # Tools don't use complete platforms
419	)
420	)
421
UNCOV 422	return UserGenerateLockfiles(out)	×
423
424
425	@dataclass(frozen=True)	7✔
426	class PythonSyntheticLockfileTargetsRequest(SyntheticTargetsRequest):	7✔
427	"""Register the type used to create synthetic targets for Python lockfiles.
428
429	As the paths for all lockfiles are known up-front, we set the `path` field to
430	`SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS` so that we get a single request for all
431	our synthetic targets rather than one request per directory.
432	"""
433
434	path: str = SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS	7✔
435
436
437	def synthetic_lockfile_target_name(resolve: str) -> str:	7✔
UNCOV 438	return f"_{resolve}_lockfile"	×
439
440
441	@rule	7✔
442	async def python_lockfile_synthetic_targets(	7✔
443	request: PythonSyntheticLockfileTargetsRequest,
444	python_setup: PythonSetup,
445	) -> SyntheticAddressMaps:
446	if not python_setup.enable_synthetic_lockfiles:	6✔
447	return SyntheticAddressMaps()	6✔
448
UNCOV 449	resolves = [	×
450	(os.path.dirname(lockfile), os.path.basename(lockfile), name)
451	for name, lockfile in python_setup.resolves.items()
452	]
453
UNCOV 454	return SyntheticAddressMaps.for_targets_request(	×
455	request,
456	[
457	(
458	os.path.join(spec_path, "BUILD.python-lockfiles"),
459	tuple(
460	TargetAdaptor(
461	"_lockfiles",
462	name=synthetic_lockfile_target_name(name),
463	sources=(lockfile,),
464	__description_of_origin__=f"the [python].resolves option {name!r}",
465	)
466	for _, lockfile, name in lockfiles
467	),
468	)
469	for spec_path, lockfiles in itertools.groupby(sorted(resolves), key=itemgetter(0))
470	],
471	)
472
473
474	def rules():	7✔
475	return (	6✔
476	*collect_rules(),
477	UnionRule(GenerateLockfile, GeneratePythonLockfile),
478	UnionRule(KnownUserResolveNamesRequest, KnownPythonUserResolveNamesRequest),
479	UnionRule(RequestedUserResolveNames, RequestedPythonUserResolveNames),
480	UnionRule(SyntheticTargetsRequest, PythonSyntheticLockfileTargetsRequest),
481	)

pantsbuild / pants / 24145945949

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous