21264706899

Committed 22 Jan 2026 09:00PM UTC coverage: 80.255% (+1.6%) from 78.666%

Build # 21264706899

Build Type

Pull #23031

github

Committed by

web-flow

Commit Message

Merge 8385604a3 into d250c80fe

Pull Request Pull Request #23031: Enable publish without package

Run Details

32 of 60 new or added lines in 6 files covered. (53.33%)

2 existing lines in 2 files now uncovered.

78788 of 98172 relevant lines covered (80.26%)

3.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.41

/src/python/pants/core/goals/package.py

# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import logging
import os
from abc import ABCMeta
from collections.abc import Iterable
from dataclasses import dataclass
from enum import StrEnum, auto
from string import Template

from pants.core.environments.rules import EnvironmentNameRequest, resolve_environment_name
from pants.core.util_rules import distdir
from pants.core.util_rules.distdir import DistDir
from pants.engine.addresses import Address
from pants.engine.environment import EnvironmentName
from pants.engine.fs import EMPTY_DIGEST, Digest, MergeDigests, Workspace
from pants.engine.goal import Goal, GoalSubsystem
from pants.engine.internals.graph import find_valid_field_sets
from pants.engine.internals.specs_rules import find_valid_field_sets_for_target_roots
from pants.engine.intrinsics import merge_digests
from pants.engine.rules import collect_rules, concurrently, goal_rule, implicitly, rule
from pants.engine.target import (
    AllTargets,
    AsyncFieldMixin,
    Dependencies,
    DepsTraversalBehavior,
    FieldSet,
    FieldSetsPerTargetRequest,
    NoApplicableTargetsBehavior,
    ShouldTraverseDepsPredicate,
    SpecialCasedDependencies,
    StringField,
    Target,
    TargetRootsToFieldSetsRequest,
    Targets,
)
from pants.engine.unions import UnionMembership, union
from pants.option.option_types import EnumOption
from pants.util.docutil import bin_name
from pants.util.logging import LogLevel
from pants.util.ordered_set import FrozenOrderedSet
from pants.util.strutil import help_text

logger = logging.getLogger(__name__)


@union(in_scope_types=[EnvironmentName])
class PackageFieldSet(FieldSet, metaclass=ABCMeta):
    """The fields necessary to build an asset from a target."""

    def has_side_effects(self) -> bool:
        """Subclasses should implement this method to check if the package request has side effects.

        Returns False by default.
        """
        return False


@dataclass(frozen=True)
class BuiltPackageArtifact:
    """Information about artifacts in a built package.

    Used for logging information about the artifacts that are dumped to the distdir.
    """

    relpath: str | None
    extra_log_lines: tuple[str, ...] = tuple()


@dataclass(frozen=True)
class BuiltPackage:
    digest: Digest
    artifacts: tuple[BuiltPackageArtifact, ...]


@rule(polymorphic=True)
async def build_package(
    package_fieldset: PackageFieldSet,
    environment_name: EnvironmentName,
) -> BuiltPackage:
    raise NotImplementedError()


class OutputPathField(StringField, AsyncFieldMixin):
    DEFAULT = "${spec_path_normalized}/${target_name_normalized}${file_suffix}"

    alias = "output_path"
    default = DEFAULT

    help = help_text(
        f"""
        Where the built asset should be located.

        This field supports the following template replacements:

        - `${{spec_path_normalized}}`: The path to the target's directory ("spec path") with forward slashes replaced by dots.

        - `${{target_name_normalized}}`: The target's name with paramaterizations escaped by replacing dots with underscores.

        - `${{file_suffix}}`: For target's which produce single file artifacts, this is the file type suffix to use with a leading dot,
          and is empty otherwise when not applicable.

        If undefined, this will use the path to the BUILD file, followed by the target name.
        For example, `src/python/project:app` would be `src.python.project/app.ext`. This behavior corresponds to
        the default template: `{DEFAULT}`

        When running `{bin_name()} package`, this path will be prefixed by `--distdir` (e.g. `dist/`).

        Warning: setting this value risks naming collisions with other package targets you may have.
        """
    )

    def parameters(self, *, file_ending: str | None) -> dict[str, str]:
        spec_path_normalized = self.address.spec_path.replace(os.sep, ".")
        if not spec_path_normalized:
            spec_path_normalized = "."

        target_name_part = (
            self.address.generated_name.replace(".", "_")
            if self.address.generated_name
            else self.address.target_name
        )
        target_params_sanitized = self.address.parameters_repr.replace(".", "_")
        target_name_normalized = f"{target_name_part}{target_params_sanitized}"

        file_suffix = ""
        if file_ending:
            assert not file_ending.startswith("."), "`file_ending` should not start with `.`"
            file_suffix = f".{file_ending}"

        return dict(
            spec_path_normalized=spec_path_normalized,
            target_name_normalized=target_name_normalized,
            file_suffix=file_suffix,
        )

    def value_or_default(self, *, file_ending: str | None) -> str:
        template_string = self.value
        assert template_string is not None
        template = Template(template_string)

        params = self.parameters(file_ending=file_ending)
        result = template.safe_substitute(params)
        return os.path.normpath(result)


class PackagingSideEffectBehavior(StrEnum):
    ALLOW = auto()
    IGNORE = auto()
    WARN = auto()
    ERROR = auto()


class SideEffectingPackageException(Exception):
    """Exception raised when a package request has unallowed side effects in the current context."""

    def __init__(self, field_set: PackageFieldSet):
        super().__init__(
            f'The package request for {field_set.address} has side effects but `[package].side_effecting_behavior` is set to "error".'
        )


@dataclass(frozen=True)
class EnvironmentAwarePackageRequest:
    """Request class to request a `BuiltPackage` in an environment-aware fashion.

    Also includes information about the context in which this package request is being called in,
    i.e. can this package request have side effects.
    """

    field_set: PackageFieldSet
    side_effecting_behavior: PackagingSideEffectBehavior | None = None


class PackageSubsystem(GoalSubsystem):
    name = "package"
    help = "Create a distributable package."

    @classmethod
    def activated(cls, union_membership: UnionMembership) -> bool:
        return PackageFieldSet in union_membership

    side_effecting_behavior = EnumOption(
        default=PackagingSideEffectBehavior.WARN,
        help="The behavior to take when a package request has side effects.",
    )


@rule
async def environment_aware_package(
    request: EnvironmentAwarePackageRequest, subsystem: PackageSubsystem
) -> BuiltPackage:
    side_effecting_behavior = request.side_effecting_behavior or subsystem.side_effecting_behavior
    match side_effecting_behavior:
        case PackagingSideEffectBehavior.IGNORE if request.field_set.has_side_effects():
            return BuiltPackage(EMPTY_DIGEST, ())
        case PackagingSideEffectBehavior.ERROR if request.field_set.has_side_effects():
            raise SideEffectingPackageException(request.field_set)
        case PackagingSideEffectBehavior.WARN if request.field_set.has_side_effects():
            logger.warning(f"Side-effecting package request for {request.field_set.address}")
    environment_name = await resolve_environment_name(
        EnvironmentNameRequest.from_field_set(request.field_set), **implicitly()
    )
    package = await build_package(
        **implicitly({request.field_set: PackageFieldSet, environment_name: EnvironmentName})
    )
    return package


class Package(Goal):
    subsystem_cls = PackageSubsystem
    environment_behavior = Goal.EnvironmentBehavior.USES_ENVIRONMENTS


class AllPackageableTargets(Targets):
    pass


@rule(desc="Find all packageable targets in project", level=LogLevel.DEBUG)
async def find_all_packageable_targets(all_targets: AllTargets) -> AllPackageableTargets:
    fs_per_target = await find_valid_field_sets(
        FieldSetsPerTargetRequest(PackageFieldSet, all_targets), **implicitly()
    )
    return AllPackageableTargets(
        target
        for target, field_sets in zip(all_targets, fs_per_target.collection)
        if len(field_sets) > 0
    )


@goal_rule
async def package_asset(workspace: Workspace, dist_dir: DistDir) -> Package:
    target_roots_to_field_sets = await find_valid_field_sets_for_target_roots(
        TargetRootsToFieldSetsRequest(
            PackageFieldSet,
            goal_description="the `package` goal",
            no_applicable_targets_behavior=NoApplicableTargetsBehavior.warn,
        ),
        **implicitly(),
    )
    if not target_roots_to_field_sets.field_sets:
        return Package(exit_code=0)

    packages = await concurrently(
        environment_aware_package(EnvironmentAwarePackageRequest(field_set), **implicitly())
        for field_set in target_roots_to_field_sets.field_sets
    )

    merged_digest = await merge_digests(MergeDigests(pkg.digest for pkg in packages))
    all_relpaths = [
        artifact.relpath for pkg in packages for artifact in pkg.artifacts if artifact.relpath
    ]

    workspace.write_digest(
        merged_digest, path_prefix=str(dist_dir.relpath), clear_paths=all_relpaths
    )
    for pkg in packages:
        for artifact in pkg.artifacts:
            msg = []
            if artifact.relpath:
                msg.append(f"Wrote {dist_dir.relpath / artifact.relpath}")
            msg.extend(str(line) for line in artifact.extra_log_lines)
            if msg:
                logger.info("\n".join(msg))
    return Package(exit_code=0)


@dataclass(frozen=True)
class TraverseIfNotPackageTarget(ShouldTraverseDepsPredicate):
    """This predicate stops dep traversal after package targets.

    When traversing deps, such as when collecting a list of transitive deps,
    this predicate effectively turns any package targets into graph leaf nodes.
    The package targets are included, but the deps of package targets are not.

    Also, this excludes dependencies from any SpecialCasedDependencies fields,
    which mirrors the behavior of the default predicate: TraverseIfDependenciesField.
    """

    package_field_set_types: FrozenOrderedSet[PackageFieldSet]
    roots: FrozenOrderedSet[Address]
    always_traverse_roots: bool  # traverse roots even if they are package targets

    def __init__(
        self,
        *,
        union_membership: UnionMembership,
        roots: Iterable[Address],
        always_traverse_roots: bool = True,
    ) -> None:
        object.__setattr__(self, "package_field_set_types", union_membership.get(PackageFieldSet))
        object.__setattr__(self, "roots", FrozenOrderedSet(roots))
        object.__setattr__(self, "always_traverse_roots", always_traverse_roots)
        super().__init__()

    def __call__(
        self, target: Target, field: Dependencies | SpecialCasedDependencies
    ) -> DepsTraversalBehavior:
        if isinstance(field, SpecialCasedDependencies):
            return DepsTraversalBehavior.EXCLUDE
        if self.always_traverse_roots and target.address in self.roots:
            return DepsTraversalBehavior.INCLUDE
        if any(
            field_set_type.is_applicable(target) for field_set_type in self.package_field_set_types
        ):
            return DepsTraversalBehavior.EXCLUDE
        return DepsTraversalBehavior.INCLUDE


def rules():
    return (*collect_rules(), *distdir.rules())

1	# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	12✔
5
6	import logging	12✔
7	import os	12✔
8	from abc import ABCMeta	12✔
9	from collections.abc import Iterable	12✔
10	from dataclasses import dataclass	12✔
11	from enum import StrEnum, auto	12✔
12	from string import Template	12✔
13
14	from pants.core.environments.rules import EnvironmentNameRequest, resolve_environment_name	12✔
15	from pants.core.util_rules import distdir	12✔
16	from pants.core.util_rules.distdir import DistDir	12✔
17	from pants.engine.addresses import Address	12✔
18	from pants.engine.environment import EnvironmentName	12✔
19	from pants.engine.fs import EMPTY_DIGEST, Digest, MergeDigests, Workspace	12✔
20	from pants.engine.goal import Goal, GoalSubsystem	12✔
21	from pants.engine.internals.graph import find_valid_field_sets	12✔
22	from pants.engine.internals.specs_rules import find_valid_field_sets_for_target_roots	12✔
23	from pants.engine.intrinsics import merge_digests	12✔
24	from pants.engine.rules import collect_rules, concurrently, goal_rule, implicitly, rule	12✔
25	from pants.engine.target import (	12✔
26	AllTargets,
27	AsyncFieldMixin,
28	Dependencies,
29	DepsTraversalBehavior,
30	FieldSet,
31	FieldSetsPerTargetRequest,
32	NoApplicableTargetsBehavior,
33	ShouldTraverseDepsPredicate,
34	SpecialCasedDependencies,
35	StringField,
36	Target,
37	TargetRootsToFieldSetsRequest,
38	Targets,
39	)
40	from pants.engine.unions import UnionMembership, union	12✔
41	from pants.option.option_types import EnumOption	12✔
42	from pants.util.docutil import bin_name	12✔
43	from pants.util.logging import LogLevel	12✔
44	from pants.util.ordered_set import FrozenOrderedSet	12✔
45	from pants.util.strutil import help_text	12✔
46
47	logger = logging.getLogger(__name__)	12✔
48
49
50	@union(in_scope_types=[EnvironmentName])	12✔
51	class PackageFieldSet(FieldSet, metaclass=ABCMeta):	12✔
52	"""The fields necessary to build an asset from a target."""
53
54	def has_side_effects(self) -> bool:	12✔
55	"""Subclasses should implement this method to check if the package request has side effects.
56
57	Returns False by default.
58	"""
NEW 59	return False	×
60
61
62	@dataclass(frozen=True)	12✔
63	class BuiltPackageArtifact:	12✔
64	"""Information about artifacts in a built package.
65
66	Used for logging information about the artifacts that are dumped to the distdir.
67	"""
68
69	relpath: str \| None	12✔
70	extra_log_lines: tuple[str, ...] = tuple()	12✔
71
72
73	@dataclass(frozen=True)	12✔
74	class BuiltPackage:	12✔
75	digest: Digest	12✔
76	artifacts: tuple[BuiltPackageArtifact, ...]	12✔
77
78
79	@rule(polymorphic=True)	12✔
80	async def build_package(	12✔
81	package_fieldset: PackageFieldSet,
82	environment_name: EnvironmentName,
83	) -> BuiltPackage:
84	raise NotImplementedError()	×
85
86
87	class OutputPathField(StringField, AsyncFieldMixin):	12✔
88	DEFAULT = "${spec_path_normalized}/${target_name_normalized}${file_suffix}"	12✔
89
90	alias = "output_path"	12✔
91	default = DEFAULT	12✔
92
93	help = help_text(	12✔
94	f"""
95	Where the built asset should be located.
96
97	This field supports the following template replacements:
98
99	- `${{spec_path_normalized}}`: The path to the target's directory ("spec path") with forward slashes replaced by dots.
100
101	- `${{target_name_normalized}}`: The target's name with paramaterizations escaped by replacing dots with underscores.
102
103	- `${{file_suffix}}`: For target's which produce single file artifacts, this is the file type suffix to use with a leading dot,
104	and is empty otherwise when not applicable.
105
106	If undefined, this will use the path to the BUILD file, followed by the target name.
107	For example, `src/python/project:app` would be `src.python.project/app.ext`. This behavior corresponds to
108	the default template: `{DEFAULT}`
109
110	When running `{bin_name()} package`, this path will be prefixed by `--distdir` (e.g. `dist/`).
111
112	Warning: setting this value risks naming collisions with other package targets you may have.
113	"""
114	)
115
116	def parameters(self, *, file_ending: str \| None) -> dict[str, str]:	12✔
117	spec_path_normalized = self.address.spec_path.replace(os.sep, ".")	6✔
118	if not spec_path_normalized:	6✔
119	spec_path_normalized = "."	1✔
120
121	target_name_part = (	6✔
122	self.address.generated_name.replace(".", "_")
123	if self.address.generated_name
124	else self.address.target_name
125	)
126	target_params_sanitized = self.address.parameters_repr.replace(".", "_")	6✔
127	target_name_normalized = f"{target_name_part}{target_params_sanitized}"	6✔
128
129	file_suffix = ""	6✔
130	if file_ending:	6✔
131	assert not file_ending.startswith("."), "`file_ending` should not start with `.`"	2✔
132	file_suffix = f".{file_ending}"	2✔
133
134	return dict(	6✔
135	spec_path_normalized=spec_path_normalized,
136	target_name_normalized=target_name_normalized,
137	file_suffix=file_suffix,
138	)
139
140	def value_or_default(self, *, file_ending: str \| None) -> str:	12✔
141	template_string = self.value	6✔
142	assert template_string is not None	6✔
143	template = Template(template_string)	6✔
144
145	params = self.parameters(file_ending=file_ending)	6✔
146	result = template.safe_substitute(params)	6✔
147	return os.path.normpath(result)	6✔
148
149
150	class PackagingSideEffectBehavior(StrEnum):	12✔
151	ALLOW = auto()	12✔
152	IGNORE = auto()	12✔
153	WARN = auto()	12✔
154	ERROR = auto()	12✔
155
156
157	class SideEffectingPackageException(Exception):	12✔
158	"""Exception raised when a package request has unallowed side effects in the current context."""
159
160	def __init__(self, field_set: PackageFieldSet):	12✔
NEW 161	super().__init__(	×
162	f'The package request for {field_set.address} has side effects but `[package].side_effecting_behavior` is set to "error".'
163	)
164
165
166	@dataclass(frozen=True)	12✔
167	class EnvironmentAwarePackageRequest:	12✔
168	"""Request class to request a `BuiltPackage` in an environment-aware fashion.
169
170	Also includes information about the context in which this package request is being called in,
171	i.e. can this package request have side effects.
172	"""
173
174	field_set: PackageFieldSet	12✔
175	side_effecting_behavior: PackagingSideEffectBehavior \| None = None	12✔
176
177
178	class PackageSubsystem(GoalSubsystem):	12✔
179	name = "package"	12✔
180	help = "Create a distributable package."	12✔
181
182	@classmethod	12✔
183	def activated(cls, union_membership: UnionMembership) -> bool:	12✔
NEW 184	return PackageFieldSet in union_membership	×
185
186	side_effecting_behavior = EnumOption(	12✔
187	default=PackagingSideEffectBehavior.WARN,
188	help="The behavior to take when a package request has side effects.",
189	)
190
191
192	@rule	12✔
193	async def environment_aware_package(	12✔
194	request: EnvironmentAwarePackageRequest, subsystem: PackageSubsystem
195	) -> BuiltPackage:
NEW 196	side_effecting_behavior = request.side_effecting_behavior or subsystem.side_effecting_behavior	×
NEW 197	match side_effecting_behavior:	×
NEW 198	case PackagingSideEffectBehavior.IGNORE if request.field_set.has_side_effects():	×
NEW 199	return BuiltPackage(EMPTY_DIGEST, ())	×
NEW 200	case PackagingSideEffectBehavior.ERROR if request.field_set.has_side_effects():	×
NEW 201	raise SideEffectingPackageException(request.field_set)	×
NEW 202	case PackagingSideEffectBehavior.WARN if request.field_set.has_side_effects():	×
NEW 203	logger.warning(f"Side-effecting package request for {request.field_set.address}")	×
UNCOV 204	environment_name = await resolve_environment_name(	×
205	EnvironmentNameRequest.from_field_set(request.field_set), **implicitly()
206	)
207	package = await build_package(	×
208	**implicitly({request.field_set: PackageFieldSet, environment_name: EnvironmentName})
209	)
210	return package	×
211
212
213	class Package(Goal):	12✔
214	subsystem_cls = PackageSubsystem	12✔
215	environment_behavior = Goal.EnvironmentBehavior.USES_ENVIRONMENTS	12✔
216
217
218	class AllPackageableTargets(Targets):	12✔
219	pass	12✔
220
221
222	@rule(desc="Find all packageable targets in project", level=LogLevel.DEBUG)	12✔
223	async def find_all_packageable_targets(all_targets: AllTargets) -> AllPackageableTargets:	12✔
224	fs_per_target = await find_valid_field_sets(	×
225	FieldSetsPerTargetRequest(PackageFieldSet, all_targets), **implicitly()
226	)
227	return AllPackageableTargets(	×
228	target
229	for target, field_sets in zip(all_targets, fs_per_target.collection)
230	if len(field_sets) > 0
231	)
232
233
234	@goal_rule	12✔
235	async def package_asset(workspace: Workspace, dist_dir: DistDir) -> Package:	12✔
236	target_roots_to_field_sets = await find_valid_field_sets_for_target_roots(	×
237	TargetRootsToFieldSetsRequest(
238	PackageFieldSet,
239	goal_description="the `package` goal",
240	no_applicable_targets_behavior=NoApplicableTargetsBehavior.warn,
241	),
242	**implicitly(),
243	)
244	if not target_roots_to_field_sets.field_sets:	×
245	return Package(exit_code=0)	×
246
247	packages = await concurrently(	×
248	environment_aware_package(EnvironmentAwarePackageRequest(field_set), **implicitly())
249	for field_set in target_roots_to_field_sets.field_sets
250	)
251
252	merged_digest = await merge_digests(MergeDigests(pkg.digest for pkg in packages))	×
253	all_relpaths = [	×
254	artifact.relpath for pkg in packages for artifact in pkg.artifacts if artifact.relpath
255	]
256
257	workspace.write_digest(	×
258	merged_digest, path_prefix=str(dist_dir.relpath), clear_paths=all_relpaths
259	)
260	for pkg in packages:	×
261	for artifact in pkg.artifacts:	×
262	msg = []	×
263	if artifact.relpath:	×
264	msg.append(f"Wrote {dist_dir.relpath / artifact.relpath}")	×
265	msg.extend(str(line) for line in artifact.extra_log_lines)	×
266	if msg:	×
267	logger.info("\n".join(msg))	×
268	return Package(exit_code=0)	×
269
270
271	@dataclass(frozen=True)	12✔
272	class TraverseIfNotPackageTarget(ShouldTraverseDepsPredicate):	12✔
273	"""This predicate stops dep traversal after package targets.
274
275	When traversing deps, such as when collecting a list of transitive deps,
276	this predicate effectively turns any package targets into graph leaf nodes.
277	The package targets are included, but the deps of package targets are not.
278
279	Also, this excludes dependencies from any SpecialCasedDependencies fields,
280	which mirrors the behavior of the default predicate: TraverseIfDependenciesField.
281	"""
282
283	package_field_set_types: FrozenOrderedSet[PackageFieldSet]	12✔
284	roots: FrozenOrderedSet[Address]	12✔
285	always_traverse_roots: bool # traverse roots even if they are package targets	12✔
286
287	def __init__(	12✔
288	self,
289	*,
290	union_membership: UnionMembership,
291	roots: Iterable[Address],
292	always_traverse_roots: bool = True,
293	) -> None:
294	object.__setattr__(self, "package_field_set_types", union_membership.get(PackageFieldSet))	1✔
295	object.__setattr__(self, "roots", FrozenOrderedSet(roots))	1✔
296	object.__setattr__(self, "always_traverse_roots", always_traverse_roots)	1✔
297	super().__init__()	1✔
298
299	def __call__(	12✔
300	self, target: Target, field: Dependencies \| SpecialCasedDependencies
301	) -> DepsTraversalBehavior:
302	if isinstance(field, SpecialCasedDependencies):	×
303	return DepsTraversalBehavior.EXCLUDE	×
304	if self.always_traverse_roots and target.address in self.roots:	×
305	return DepsTraversalBehavior.INCLUDE	×
306	if any(	×
307	field_set_type.is_applicable(target) for field_set_type in self.package_field_set_types
308	):
309	return DepsTraversalBehavior.EXCLUDE	×
310	return DepsTraversalBehavior.INCLUDE	×
311
312
313	def rules():	12✔
314	return (collect_rules(), distdir.rules())	12✔

pantsbuild / pants / 21264706899

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous