pantsbuild / pants / 19015773527

# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).

# Licensed under the Apache License, Version 2.0 (see LICENSE).

    ConstraintParseError,

    ConstraintSatisfied,

    ConstraintsList,

)

    ExternalToolError,

    ExternalToolRequest,

    download_external_tool,

)

    CpBinary,

    MkdirBinary,

    MvBinary,

    RmBinary,

    ShBinary,

    TestBinary,

)

    version_and_tag: str,

) -> tuple[Version | None, Version | None]:

            f"Unable to parse the provided URL since it does not have a cpython prefix as per the PBS naming convention: {url}"

        )

            f"Unable to parse the provided URL because it does not follow the PBS naming convention: {url}"

        )

            "Unable to parse the Python version and PBS release tag from the provided URL "

            f"because it does not follow the PBS naming convention: {url}"

        )

    platform: Platform

            "x86_64" | "x86_64_v2" | "x86_64_v3" | "x86_64_v4",

            "unknown",

            "linux",

            "gnu" | "musl",

            *_,

        ]:

                "Unable to parse the platform from the provided URL "

                f"because it does not follow the PBS naming convention: {url}"

            )

            f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, "

            f"the value `{orig_value}` could not be parsed: {e}"

        )

        py_version=py_version,

        pbs_release_tag=pbs_release_tag,

        platform=platform,

        url=url,

        sha256=sha256,

        size=int(size),

    )

            py_version_and_tag_str

        )

            f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, "

            f"the value `{orig_value}` declares version `{py_version_and_tag_str}` in the first field, "

            "but it could not be parsed as a PBS release version."

        )

        Platform.linux_x86_64.value,

        Platform.linux_arm64.value,

        Platform.macos_x86_64.value,

        Platform.macos_arm64.value,

    ):

    else:

            f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, "

            f"the value `{orig_value}` declares platforn `{platform_str}` in the second field, "

            "but that value is not a known Pants platform. It must be one of "

            "`linux_x86_64`, `linux_arm64`, `macos_x86_64`, or `macos_arm64`."

        )

            f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, "

            f"the value `{orig_value}` declares SHA256 checksum `{sha256}` in the third field, "

            "but that value does not parse as a SHA256 checksum."

        )

            f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, "

            f"the value `{orig_value}` declares file size `{filesize_str}` in the fourth field, "

            "but that value does not parse as an integer."

        )

            maybe_inferred_py_version,

            maybe_inferred_pbs_release_tag,

            maybe_inferred_platform,

        ) = _parse_pbs_url(url)

        *, explicit: _T | None, inferred: _T | None, description: str, field_pos: str

    ) -> _T:

                    f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, "

                    f"the value `{orig_value}` does not declare a {description} in the {field_pos} field, and no {description} "

                    "could be inferred from the URL."

                )

            else:

        else:

                    f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, "

                    f"the value `{orig_value}` declares {description} `{explicit}` in the {field_pos} field, but Pants inferred "

                    f"{description} `{inferred}` from the URL."

                )

        explicit=maybe_py_version,

        inferred=maybe_inferred_py_version,

        description="version",

        field_pos="first",

    )

        explicit=maybe_pbs_release_tag,

        inferred=maybe_inferred_pbs_release_tag,

        description="PBS release tag",

        field_pos="first",

    )

        explicit=maybe_platform,

        inferred=maybe_inferred_platform,

        description="platform",

        field_pos="second",

    )

        py_version=maybe_py_version,

        pbs_release_tag=maybe_pbs_release_tag,

        platform=maybe_platform,

        url=url,

        sha256=sha256,

        size=filesize,

    )

        """

        A subsystem for Pants-provided Python leveraging Python Build Standalone (or PBS) (https://gregoryszorc.com/docs/python-build-standalone/main/).

        Enabling this subsystem will switch Pants from trying to find an appropriate Python on your

        system to using PBS to download the correct Python(s).

        The Pythons provided by PBS will be used to run any "user" code (your Python code as well

        as any Python-based tools you use, like black or pylint). The Pythons are also read-only to

        ensure they remain hermetic across runs of different tools and code.

        The Pythons themselves are stored in your `named_caches_dir`: https://www.pantsbuild.org/docs/reference-global#named_caches_dir

        under `python_build_standalone/<version>`. Wiping the relevant version directory

        (with `sudo rm -rf`) will force a re-download of Python.

        WARNING: PBS does have some behavior quirks, most notably that it has some hardcoded references

        to build-time paths (such as constants that are found in the `sysconfig` module). These paths

        may be used when trying to compile some extension modules from source.

        For more info, see https://gregoryszorc.com/docs/python-build-standalone/main/quirks.html.

        """

    )

        default=None,

        default_help_repr=f"<Metadata for versions: {', '.join(sorted(load_pbs_pythons()))}>",

        advanced=True,

        help=textwrap.dedent(

            f"""

            Known versions to verify downloads against.

            Each element is a pipe-separated string of either `py_version+pbs_release_tag|platform|sha256|length|url` or

            `sha256|length|url`, where:

            - `py_version` is the Python version string

            - `pbs_release_tag` is the PBS release tag (i.e., the PBS-specific version)

            - `platform` is one of `[{",".join(Platform.__members__.keys())}]`

            - `sha256` is the 64-character hex representation of the expected sha256

                digest of the download file, as emitted by `shasum -a 256`

            - `length` is the expected length of the download file in bytes, as emitted by

                `wc -c`

            - `url` is the download URL to the `.tar.gz` archive

            E.g., `3.1.2|macos_x86_64|6d0f18cd84b918c7b3edd0203e75569e0c7caecb1367bbbe409b44e28514f5be|42813|https://<URL>`

            or `https://<URL>|6d0f18cd84b918c7b3edd0203e75569e0c7caecb1367bbbe409b44e28514f5be|42813`.

            Values are space-stripped, so pipes can be indented for readability if necessary. If the three field

            format is used, then Pants will infer the `py_version`, `pbs_release_tag`, and `platform` fields from

            the URL. With the five field format, one or more of `py_version`, `pbs_release_tag`, and `platform`

            may be left blank if Pants can infer the field from the URL.

            Additionally, any versions you specify here will override the default Pants metadata for

            that version.

            """

        ),

    )

        default=None,

        help=textwrap.dedent(

            """

            Version constraints on the PBS "release" version to ensure only matching PBS releases are considered.

            Constraints should be specfied using operators like `>=`, `<=`, `>`, `<`, `==`, or `!=` in a similar

            manner to Python interpreter constraints: e.g., `>=20241201` or `>=20241201,<20250101`.

            """

        ),

    )

                f"The `[{PBSPythonProviderSubsystem.options_scope}].release_constraints option` is not valid: {e}"

            ) from None

                    f"Each value for the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option "

                    "must be a set of three or five values separated by `|` characters as follows:\n\n"

                    "- 3 fields: URL|SHA256|FILE_SIZE\n\n"

                    "- 5 fields: PYTHON_VERSION+PBS_RELEASE|PLATFORM|SHA256|FILE_SIZE|URL\n\n"

                    "\n\nIf 3 fields are provided, Pants will attempt to infer values based on the URL which must "

                    "follow the PBS naming conventions.\n\n"

                    f"Instead, the following value was provided: {version_info}"

                )

                _parse_from_three_fields(version_parts, orig_value=version_info)

                if len(version_parts) == 3

                else _parse_from_five_fields(version_parts, orig_value=version_info)

            )

                pbs_python_info

            )

                release_tag,

                platform_metadata_for_releases,

            ) in release_metadatas_for_py_version.items():

    interpreter_constraints: InterpreterConstraints,

    universe: Iterable[str],

    pbs_versions: Mapping[str, Mapping[str, Mapping[str, PBSPythonInfo]]],

    platform: Platform,

    release_constraints: ConstraintSatisfied,

) -> tuple[str, Version, PBSPythonInfo]:

    """Choose the highest supported patchlevel of the lowest supported major/minor version

    consistent with any PBS release constraint."""

    # Construct a list of candidate PBS releases.

            softwrap(

                f"""\

                Failed to find a supported Python Build Standalone for Interpreter Constraint: {interpreter_constraints.description}

                Supported versions are currently: {sorted(pbs_versions)}.

                You can teach Pants about newer Python versions supported by Python Build Standalone

                by setting the `known_python_versions` option in the {PBSPythonProviderSubsystem.name}

                subsystem. Run `{bin_name()} help-advanced {PBSPythonProviderSubsystem.options_scope}`

                for more info.

                """

            )

        )

    # Choose the highest supported patchlevel of the lowest supported major/minor version

    # by searching until the major/minor version increases or the search ends (in which case the

    # last candidate is the one).

    #

    # This also sorts by release tag in ascending order. So it chooses the highest available PBS

    # release for that chosen Python version.

            # Last candidate, we're good!

            i == len(candidate_pbs_releases) - 1

            # Next candidate is the next major/minor version, so this is the highest patchlevel.

            or candidate_pbs_releases[i + 1][0][0:2] != py_version_triplet[0:2]

        ):

    request: PBSPythonProvider,

    python_setup: PythonSetup,

    pbs_subsystem: PBSPythonProviderSubsystem,

    platform: Platform,

    named_caches_dir: NamedCachesDirOption,

    sh: ShBinary,

    mkdir: MkdirBinary,

    cp: CpBinary,

    mv: MvBinary,

    rm: RmBinary,

    test: TestBinary,

) -> PythonExecutable:

        request.interpreter_constraints,

        python_setup.interpreter_versions_universe,

        versions_info,

        platform,

        pbs_subsystem.release_constraints,

    )

        ExternalToolRequest(

            DownloadFile(

                pbs_py_info["url"],

                FileDigest(

                    pbs_py_info["sha256"],

                    pbs_py_info["size"],

                ),

            ),

            "python/bin/python3",

        )

    )

    # The final desired destination within the named cache:

    # Temporary directory (on the same filesystem as the persisted destination) to copy to,

    # incorporating uniqueness so that we don't collide with concurrent invocations

        **implicitly(

            Process(

                [

                    sh.path,

                    "-euc",

                    # Atomically-copy the downloaded files into the named cache, in 5 steps:

                    #

                    # 1. Check if the target directory already exists, skipping all the work if it does

                    #    (the atomic creation means this will be fully created by an earlier execution,

                    #    no torn state).

                    #

                    # 2. Copy the files into a temporary directory within the persistent named cache. Copying

                    #    into a temporary directory ensures that we don't end up with partial state if this

                    #    process is interrupted. Placing the temporary directory within the persistent named

                    #    cache ensures it is on the same filesystem as the final destination, allowing for

                    #    atomic mv.

                    #

                    # 3. Actually move the temporary directory to the final destination, failing if it

                    #    already exists (which would indicate a concurrent execution), but squashing that

                    #    error. Note that this is specifically moving to the parent directory of the target,

                    #    i.e. something like:

                    #

                    #        mv .python_build_standalone/tmp/pbs-copier-.../3.10.11 .python_build_standalone

                    #

                    #    which detects `.python_build_standalone` is a directory and thus attempts to create

                    #    `.python_build_standalone/3.10.11`. This fails if that target already exists. The

                    #    alternative of explicitly passing the final target like

                    #    `mv ... .python_build_standalone/3.10.11` will (incorrectly) create nested

                    #    directory `.python_build_standalone/3.10.11/3.10.11` if the target already exists.

                    #

                    # 4. Check it worked. In particular, mv might fail for a different reason than the final

                    #    destination already existing: in those cases, we won't have put things in the right

                    #    place, and downstream code won't have the Python it needs. So, we check that the

                    #    final destination exists and fail if it doesn't, surfacing any errors to the user.

                    #

                    # 5. Clean-up the temporary files

                    f"""

                    # Step 1: check and skip

                    if {test.path} -d {persisted_destination}; then

                        echo "{persisted_destination} already exists, fully created by earlier execution" >&2

                        exit 0

                    fi

                    # Step 2: copy from the digest into the named cache

                    {mkdir.path} -p "{temp_dir}"

                    {cp.path} -R python "{copy_target}"

                    # Step 3: attempt to move, squashing the error

                    {mv.path} "{copy_target}" "{persisted_destination.parent}" || echo "mv failed: $?" >&2

                    # Step 4: confirm and clean-up

                    if ! {test.path} -d "{persisted_destination}"; then

                        echo "Failed to create {persisted_destination}" >&2

                        exit 1

                    fi

                    # Step 5: remove the temporary directory

                    {rm.path} -rf "{temp_dir}"

                    """,

                ],

                level=LogLevel.DEBUG,

                input_digest=downloaded_python.digest,

                description=f"Install Python {python_version}",

                append_only_caches=PBS_APPEND_ONLY_CACHES,

                # Don't cache, we want this to always be run so that we can assume for the rest of the

                # session the named_cache destination for this Python is valid, as the Python ecosystem

                # mainly assumes absolute paths for Python interpreters.

                cache_scope=ProcessCacheScope.PER_SESSION,

            ),

        ),

    )

        path=str(python_path),

        fingerprint=None,

        # One would normally set append_only_caches=PBS_APPEND_ONLY_CACHES

        # here, but it is already going to be injected into the pex

        # environment by PythonBuildStandaloneBinary

    )

        *collect_rules(),

        *pex_rules(),

        *external_tools_rules(),

        UnionRule(PythonProvider, PBSPythonProvider),

    )