pantsbuild / pants / 19015773527

# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).

# Licensed under the Apache License, Version 2.0 (see LICENSE).

    AddPrefix,

    CreateDigest,

    Digest,

    DigestSubset,

    FileContent,

    FileDigest,

    MergeDigests,

    PathGlobs,

    RemovePrefix,

    Snapshot,

)

    create_digest,

    digest_subset_to_digest,

    digest_to_snapshot,

    get_digest_contents,

    merge_digests,

    path_globs_to_digest,

    remove_prefix,

)

    ClasspathEntry,

    ClasspathEntryRequest,

    CompileResult,

    FallibleClasspathEntry,

)

    ArtifactRequirement,

    ArtifactRequirements,

    GatherJvmCoordinatesRequest,

)

    JvmArtifactFieldSet,

    JvmArtifactJarSourceField,

    JvmArtifactTarget,

    JvmResolveField,

)

if TYPE_CHECKING:

    from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool

    """An exception relating to invoking Coursier or processing its output."""

    """No compatible resolve could be found for a set of targets."""

            f"{resolve}:\n{bullet_list(sorted(addresses))}"

            for resolve, addresses in sorted(resolves_to_addresses.items())

        )

            f"{msg_prefix}:\n\n"

            f"{formatted_resolve_lists}\n\n"

            "Targets which will be merged onto the same classpath must share a resolve (from the "

            f"[resolve]({doc_url('reference/targets/deploy_jar#resolve')}) field)."

        )

    """A single artifact entry from a Coursier-resolved lockfile.

    These fields are nearly identical to the JSON objects from the

    "dependencies" entries in Coursier's --json-output-file format.

    But unlike Coursier's JSON report, a CoursierLockfileEntry

    includes the content-address of the artifact fetched by Coursier

    and ingested by Pants.

    For example, a Coursier JSON report dependency entry might look like this:

    ```

    {

      "coord": "com.chuusai:shapeless_2.13:2.3.3",

      "file": "/home/USER/.cache/coursier/v1/https/repo1.maven.org/maven2/com/chuusai/shapeless_2.13/2.3.3/shapeless_2.13-2.3.3.jar",

      "directDependencies": [

        "org.scala-lang:scala-library:2.13.0"

      ],

      "dependencies": [

        "org.scala-lang:scala-library:2.13.0"

      ]

    }

    ```

    The equivalent CoursierLockfileEntry would look like this:

    ```

    CoursierLockfileEntry(

        coord="com.chuusai:shapeless_2.13:2.3.3", # identical

        file_name="shapeless_2.13-2.3.3.jar" # PurePath(entry["file"].name)

        direct_dependencies=(Coordinate.from_coord_str("org.scala-lang:scala-library:2.13.0"),),

        dependencies=(Coordinate.from_coord_str("org.scala-lang:scala-library:2.13.0"),),

        file_digest=FileDigest(fingerprint=<sha256 of the jar>, ...),

    )

    ```

    The fields `remote_url` and `pants_address` are set by Pants if the `coord` field matches a

    `jvm_artifact` that had either the `url` or `jar` fields set.

    """

        """Construct a CoursierLockfileEntry from its JSON dictionary representation."""

            coord=Coordinate.from_json_dict(entry["coord"]),

            file_name=entry["file_name"],

            direct_dependencies=Coordinates(

                Coordinate.from_json_dict(d) for d in entry["directDependencies"]

            ),

            dependencies=Coordinates(Coordinate.from_json_dict(d) for d in entry["dependencies"]),

            file_digest=FileDigest(

                fingerprint=entry["file_digest"]["fingerprint"],

                serialized_bytes_length=entry["file_digest"]["serialized_bytes_length"],

            ),

            remote_url=entry.get("remote_url"),

            pants_address=entry.get("pants_address"),

        )

        """Export this CoursierLockfileEntry to a JSON object."""

            coord=self.coord.to_json_dict(),

            directDependencies=[coord.to_json_dict() for coord in self.direct_dependencies],

            dependencies=[coord.to_json_dict() for coord in self.dependencies],

            file_name=self.file_name,

            file_digest=dict(

                fingerprint=self.file_digest.fingerprint,

                serialized_bytes_length=self.file_digest.serialized_bytes_length,

            ),

            remote_url=self.remote_url,

            pants_address=self.pants_address,

        )

    """An in-memory representation of Pants' Coursier lockfile format.

    All coordinates in the resolved lockfile will be compatible, so we do not need to do version

    testing when looking up coordinates.

    """

        # TODO: After fixing https://github.com/pantsbuild/pants/issues/13496, coordinate matches

        # should become exact, and this error message will capture all cases of stale lockfiles.

            f"{coord} was not present in resolve `{key.name}` at `{key.path}`.\n"

            f"If you have recently added new `{JvmArtifactTarget.alias}` targets, you might "

            f"need to update your lockfile by running `coursier-resolve --names={key.name}`."

        )

        self, key: CoursierResolveKey, coord: Coordinate

    ) -> tuple[CoursierLockfileEntry, tuple[CoursierLockfileEntry, ...]]:

        """Return the entry for the given Coordinate, and for its direct dependencies."""

            entry,

            tuple(entries[(i.group, i.artifact, i.classifier)] for i in entry.direct_dependencies),

        )

        self, key: CoursierResolveKey, coord: Coordinate

    ) -> tuple[CoursierLockfileEntry, tuple[CoursierLockfileEntry, ...]]:

        """Return the entry for the given Coordinate, and for its transitive dependencies."""

            entry,

            tuple(

                dependency_entry

                for d in entry.dependencies

                # The dependency might not be present in the entries due to coursier bug:

                # https://github.com/coursier/coursier/issues/2884

                # As a workaround, if this happens, we want to skip the dependency.

                # TODO Drop the check once the bug is fixed.

                if (dependency_entry := entries.get((d.group, d.artifact, d.classifier)))

                is not None

            ),

        )

        """Constructs a CoursierResolvedLockfile from it's TOML + metadata comment representation.

        The toml file should consist of an `[entries]` block, followed by several entries.

        """

        lockfile_str: str

        lockfile_bytes: bytes

        else:

            CoursierLockfileEntry.from_json_dict(entry) for entry in (contents["entries"])

        )

            entries=entries,

            metadata=metadata,

        )

        """Construct a CoursierResolvedLockfile from its serialized representation (either TOML with

        attached metadata, or old-style JSON.)."""

        """Export this CoursierResolvedLockfile to a human-readable serialized form.

        This serialized form is intended to be checked in to the user's repo as a hermetic snapshot

        of a Coursier resolved JVM classpath.

        """

            "entries": [entry.to_json_dict() for entry in self.entries],

        }

    """Calculates the destination filename on the classpath for the given source filename and coord.

    TODO: This is duplicated in `COURSIER_POST_PROCESSING_SCRIPT`.

    """

        """Return coursier arguments that can be used to compute or fetch this resolve.

        Must be used in concert with `digest`.

        """

            self.coord_arg_strings,

            itertools.chain.from_iterable(

                zip(itertools.repeat("--force-version"), self.force_version_coord_arg_strings)

            ),

            self.extra_args,

        )

    artifact_requirements: ArtifactRequirements,

) -> CoursierResolveInfo:

    # Transform requirements that correspond to local JAR files into coordinates with `file:/`

    # URLs, and put the files in the place specified by the URLs.

        else:

        (req.coordinate, exclude)

        for req in artifact_requirements

        for exclude in (req.excludes or [])

    ]

            LOCAL_EXCLUDE_FILE,

            "\n".join(

                f"{coord.group}:{coord.artifact}--{exclude}" for (coord, exclude) in excludes

            ).encode("utf-8"),

        )

        determine_source_files(SourceFilesRequest([jar_source_field]))

        for _, jar_source_field in jars

    )

        dataclasses.replace(

            req, jar=None, url=f"file:{Coursier.working_directory_placeholder}/{path}"

        )

        for (req, _), path in zip(jars, jar_file_paths)

    ]

    # Coursier only fetches non-jar artifact types ("packaging" in Pants parlance) if passed an `-A` option

    # explicitly requesting that the non-jar artifact(s) be fetched. This is an addition to passing the coordinate

    # with the desired type (packaging) value.

        # Note: `-A` defaults to `jar,bundle` and any value set replaces (and does not supplement) those defaults,

        # so the defaults must be included here for them to remain usable.

        MergeDigests(

            [

                *(jar_file_source.snapshot.digest for jar_file_source in jar_file_sources),

                excludes_digest,

            ]

        )

    )

        coord_arg_strings=FrozenOrderedSet(coord_arg_strings),

        force_version_coord_arg_strings=FrozenOrderedSet(force_version_coord_arg_strings),

        digest=digest,

        extra_args=tuple(extra_args),

    )

    artifact_requirements: ArtifactRequirements,

) -> CoursierResolvedLockfile:

    """Run `coursier fetch ...` against a list of Maven coordinates and capture the result.

    This rule does two things in a single Process invocation:

        * Runs `coursier fetch` to let Coursier do the heavy lifting of resolving

          dependencies and downloading resolved artifacts (jars, etc).

        * Copies the resolved artifacts into the Process output directory, capturing

          the artifacts as content-addressed `Digest`s.

    It's important that this happens in the same process, since the process isn't

    guaranteed to run on the same machine as the rule, nor is a subsequent process

    invocation.  This guarantees that whatever Coursier resolved, it was fully

    captured into Pants' content addressed artifact storage.

    Note however that we still get the benefit of Coursier's "global" cache if it

    had already been run on the machine where the `coursier fetch` runs, so rerunning

    `coursier fetch` tends to be fast in practice.

    Finally, this rule bundles up the result into a `CoursierResolvedLockfile`.  This

    data structure encapsulates everything necessary to either materialize the

    resolved dependencies to a classpath for Java invocations, or to write the

    lockfile out to the workspace to hermetically freeze the result of the resolve.

    """

        **implicitly(

            CoursierFetchProcess(

                args=(

                    coursier_report_file_name,

                    *coursier_resolve_info.argv,

                ),

                input_digest=coursier_resolve_info.digest,

                output_directories=("classpath",),

                output_files=(coursier_report_file_name,),

                description=(

                    "Running `coursier fetch` against "

                    f"{pluralize(len(artifact_requirements), 'requirement')}: "

                    f"{', '.join(req.to_coord_arg_str() for req in artifact_requirements)}"

                ),

            )

        )

    )

        DigestSubset(process_result.output_digest, PathGlobs([coursier_report_file_name]))

    )

        classpath_dest_filename(dep["coord"], dep["file"]) for dep in report["dependencies"]

    )

        digest_subset_to_digest(

            DigestSubset(process_result.output_digest, PathGlobs([output_path]))

        )

        for output_path in artifact_output_paths

    )

        remove_prefix(RemovePrefix(artifact_digest, "classpath"))

        for artifact_digest in artifact_digests

    )

        digest_to_file_digest(ExtractFileDigest(stripped_artifact_digest, file_name))

        for stripped_artifact_digest, file_name in zip(

            stripped_artifact_digests, artifact_file_names

        )

    )

        entries=tuple(

            CoursierLockfileEntry(

                coord=Coordinate.from_coord_str(dep["coord"]),

                direct_dependencies=Coordinates(

                    Coordinate.from_coord_str(dd) for dd in dep["directDependencies"]

                ),

                dependencies=Coordinates(Coordinate.from_coord_str(d) for d in dep["dependencies"]),

                file_name=file_name,

                file_digest=artifact_file_digest,

            )

            for dep, file_name, artifact_file_digest in zip(

                report["dependencies"], artifact_file_names, artifact_file_digests

            )

        )

    )

    coursier_resolve: CoursierResolveKey,

) -> CoursierResolvedLockfile:

    """A collection of resolved classpath entries."""

    request: CoursierLockfileEntry,

) -> ClasspathEntry:

    """Run `coursier fetch --intransitive` to fetch a single artifact.

    This rule exists to permit efficient subsetting of a "global" classpath

    in the form of a lockfile.  Callers can determine what subset of dependencies

    from the lockfile are needed for a given target, then request those

    lockfile entries individually.

    By fetching only one entry at a time, we maximize our cache efficiency.  If instead

    we fetched the entire subset that the caller wanted, there would be a different cache

    key for every possible subset.

    This rule also guarantees exact reproducibility.  If all caches have been

    removed, `coursier fetch` will re-download the artifact, and this rule will

    confirm that what was downloaded matches exactly (by content digest) what

    was specified in the lockfile (what Coursier originally downloaded).

    """

    # Prepare any URL- or JAR-specifying entries for use with Coursier

    req: ArtifactRequirement

            **implicitly(

                UnparsedAddressInputs(

                    [request.pants_address],

                    owning_address=None,

                    description_of_origin="<infallible - coursier fetch>",

                )

            )

        )

    else:

        **implicitly(

            CoursierFetchProcess(

                args=(

                    coursier_report_file_name,

                    "--intransitive",

                    *coursier_resolve_info.argv,

                ),

                input_digest=coursier_resolve_info.digest,

                output_directories=("classpath",),

                output_files=(coursier_report_file_name,),

                description=f"Fetching with coursier: {request.coord.to_coord_str()}",

            )

        )

    )

        DigestSubset(process_result.output_digest, PathGlobs([coursier_report_file_name]))

    )

            "Coursier fetch report has multiple dependencies, but exactly 1 was expected."

        )

            f'Coursier resolved coord "{resolved_coord.to_coord_str()}" does not match requested coord "{request.coord.to_coord_str()}".'

        )

        DigestSubset(process_result.output_digest, PathGlobs([classpath_dest]))

    )

        ExtractFileDigest(stripped_digest, classpath_dest_name)

    )

            f"Coursier fetch for '{resolved_coord}' succeeded, but fetched artifact {file_digest} did not match the expected artifact: {request.file_digest}."

        )

    # TODO: Loading this per JvmArtifact.

        [requirement], LockfileContext.USER

    ):

            f"Requirement `{requirement.to_coord_arg_str()}` has changed since the lockfile "

            f"for {request.resolve.path} was generated. Run `{bin_name()} generate-lockfiles` to update your "

            "lockfile based on the new requirements."

        )

    # All of the transitive dependencies are exported.

    # TODO: Expose an option to control whether this exports only the root, direct dependencies,

    # transitive dependencies, etc.

        request.resolve,

        requirement.coordinate,

    )

        coursier_fetch_one_coord(entry) for entry in (root_entry, *transitive_entries)

    )

        description=str(request.component),

        result=CompileResult.SUCCEEDED,

        output=ClasspathEntry.merge(exported_digest, classpath_entries),

        exit_code=0,

    )

    """Fetch every artifact in a lockfile."""

        coursier_fetch_one_coord(entry) for entry in lockfile.entries

    )

    coarsened_targets: CoarsenedTargets, jvm: JvmSubsystem

) -> CoursierResolveKey:

    """Selects and validates (transitively) a single resolve for a set of roots in a compile graph.

    In most cases, a `CoursierResolveKey` should be requested for a single `CoarsenedTarget` root,

    which avoids coupling un-related roots unnecessarily. But in other cases, a single compatible

    resolve is required for multiple roots (such as when running a `repl` over unrelated code), and

    in that case there might be multiple CoarsenedTargets.

    """

    # Find a single resolve that is compatible with all targets in the closure.

            jvm, "The selected targets did not have a resolve in common", targets

        )

    # Load the resolve.

        [resolve_path],

        glob_match_error_behavior=GlobMatchErrorBehavior.error,

        description_of_origin=f"The resolve `{resolve}` from `[jvm].resolves`",

    )

    """A request to set up the classpath for a JVM tool by fetching artifacts and merging the

    classpath.

    :param prefix: if set, should be a relative directory that will

        be prepended to every classpath element.  This is useful for

        keeping all classpath elements isolated under a single directory

        in a process invocation, where other inputs on the process's

        root directory might interfere with un-prefixed classpath

        entries (or vice versa).

    """

                f"Exactly one of `lockfile` or `artifact_requirements` must be provided: {self}"

            )

    """A fully fetched and merged classpath for running a JVM tool."""

        """Returns optionally prefixed classpath entry filenames.

        :param prefix: if set, will be prepended to all entries.  This is useful

            if the process working directory is not the same as the root

            directory for the process input `Digest`.

        """

    else:

                importlib.resources.files(lockfile_req.default_lockfile_resource[0])

                .joinpath(lockfile_req.default_lockfile_resource[1])

                .read_bytes()

            )

        else:

                **implicitly(PathGlobs([lockfile_req.lockfile]))

            )

                    f"No lockfile found at {lockfile_req.lockfile}, which is configured "

                    f"by the option {lockfile_req.lockfile_option_name}."

                    f"Run {regen_command} to generate it."

                )

                CoursierResolveKey(

                    name=lockfile_req.resolve_name,

                    path=lockfile_req.lockfile,

                    digest=lockfile_snapshot.digest,

                )

            )

        # Validate that the lockfile is correct.

            GatherJvmCoordinatesRequest(

                lockfile_req.artifact_inputs, lockfile_req.artifact_option_name

            )

        )

            lockfile_inputs, LockfileContext.TOOL

        ):

                f"The lockfile {lockfile_req.lockfile} (configured by the option "

                f"{lockfile_req.lockfile_option_name}) was generated with different requirements "

                f"than are currently set via {lockfile_req.artifact_option_name}. Run "

                f"{regen_command} to regenerate the lockfile."

            )

        **implicitly(MergeDigests(classpath_entry.digest for classpath_entry in classpath_entries))

    )

            **implicitly(AddPrefix(merged_snapshot.digest, request.prefix))

        )

        *collect_rules(),

        *coursier_setup.rules(),

        UnionRule(ClasspathEntryRequest, CoursierFetchRequest),

    ]