25357472185

Committed 05 May 2026 04:13AM UTC coverage: 92.941% (-0.02%) from 92.956%

Build # 25357472185

Build Type

push

github

Committed by

web-flow

Commit Message

Better change detection on deleted files (#23311)

There are several longstanding issues on this:

https://github.com/pantsbuild/pants/issues/13232
https://github.com/pantsbuild/pants/issues/14975
https://github.com/pantsbuild/pants/issues/17512
https://github.com/pantsbuild/pants/issues/23240

This has historically been intractable because the
design of dep inference and of change detection
are at odds with each other.

A fully robust solution is still somewhat out of reach,
but this change provides a general-purpose mechanism
that backends can opt in to, and uses that mechanism
in the python backend.

The result is a substantial yet incremental improvement
in the most prominent use case.

The limitation of this mechanism is that it can only
be used if a backend can infer a likely dependency on
a file simply from its path. In Python this is possible because
dotted module paths must recapitulate the filesystem path
from the source root. But this is not true in general for
all languages.

Coverage Stats

56 of 78 new or added lines in 8 files covered. (71.79%)

4 existing lines in 3 files now uncovered.

91907 of 98887 relevant lines covered (92.94%)

4.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.27

/src/python/pants/engine/internals/mapper.py

# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import annotations

import os.path
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from typing import TypeVar

from pants.backend.project_info.filter_targets import FilterSubsystem
from pants.base.exceptions import MappingError
from pants.build_graph.address import Address, BuildFileAddress
from pants.engine.addresses import Addresses
from pants.engine.collection import Collection
from pants.engine.env_vars import EnvironmentVars
from pants.engine.internals.defaults import BuildFileDefaults, BuildFileDefaultsParserState
from pants.engine.internals.dep_rules import (
    BuildFileDependencyRules,
    BuildFileDependencyRulesParserState,
)
from pants.engine.internals.parser import BuildFilePreludeSymbols, Parser
from pants.engine.internals.target_adaptor import TargetAdaptor
from pants.engine.target import RegisteredTargetTypes, Tags, Target
from pants.util.filtering import TargetFilter, and_filters, create_filters
from pants.util.frozendict import FrozenDict
from pants.util.memo import memoized_property


class DuplicateNameError(MappingError):
    """Indicates more than one top-level object was found with the same name."""


AddressMapT = TypeVar("AddressMapT", bound="AddressMap")


# A pseudo-spec to refer to deleted paths.
DELETED_SPEC_PATH = "__deleted"
# A pseudo-target type to refer to deleted paths.
DELETED_TARGET_TYPE = "__deleted_target"


@dataclass(frozen=True)
class AddressMap:
    """Maps target adaptors from a byte source."""

    path: str
    name_to_target_adaptor: FrozenDict[str, TargetAdaptor]

    @classmethod
    def parse(
        cls,
        filepath: str,
        build_file_content: str,
        parser: Parser,
        extra_symbols: BuildFilePreludeSymbols,
        env_vars: EnvironmentVars,
        is_bootstrap: bool,
        defaults: BuildFileDefaultsParserState,
        dependents_rules: BuildFileDependencyRulesParserState | None,
        dependencies_rules: BuildFileDependencyRulesParserState | None,
    ) -> AddressMap:
        """Parses a source for targets.

        The target adaptors are all 'thin': any targets they point to in other namespaces or even in
        the same namespace but from a separate source are left as unresolved pointers.
        """
        try:
            target_adaptors = parser.parse(
                filepath,
                build_file_content,
                extra_symbols,
                env_vars,
                is_bootstrap,
                defaults,
                dependents_rules,
                dependencies_rules,
            )
        except Exception as e:
            raise MappingError(f"Failed to parse ./{filepath}:\n{type(e).__name__}: {e}")
        return cls.create(filepath, target_adaptors)

    @classmethod
    def create(
        cls: type[AddressMapT], filepath: str, target_adaptors: Iterable[TargetAdaptor]
    ) -> AddressMapT:
        name_to_target_adaptors: dict[str, TargetAdaptor] = {}
        for target_adaptor in target_adaptors:
            name = target_adaptor.name or os.path.basename(os.path.dirname(filepath))
            if name in name_to_target_adaptors:
                duplicate = name_to_target_adaptors[name]
                raise DuplicateNameError(
                    f"A target already exists at {filepath!r} with name {name!r} and target type "
                    f"{duplicate.type_alias!r}. The {target_adaptor.type_alias!r} target "
                    "cannot use the same name."
                )
            name_to_target_adaptors[name] = target_adaptor
        return cls(filepath, FrozenDict(sorted(name_to_target_adaptors.items())))


class DifferingFamiliesError(MappingError):
    """Indicates an attempt was made to merge address maps from different families together."""


@dataclass(frozen=True)
class AddressFamily:
    """Represents the family of target adaptors collected from the BUILD files in one directory.

    To create an AddressFamily, use `create`.

    An address family can be composed of the target adaptors from zero or more underlying address
    sources. An "empty" AddressFamily is legal, and is the result when there are not build files in
    a particular namespace.

    :param namespace: The namespace path of this address family.
    :param name_to_target_adaptors: A dict mapping from name to the target adaptor.
    :param defaults: The default target field values, per target type, applicable for this address family.
    :param dependents_rules: The rules to apply on incoming dependencies to targets in this family.
    :param dependencies_rules: The rules to apply on the outgoing dependencies from targets in this family.
    """

    # The directory from which the adaptors were parsed.
    namespace: str
    name_to_target_adaptors: dict[str, tuple[str, TargetAdaptor]]
    defaults: BuildFileDefaults
    dependents_rules: BuildFileDependencyRules | None
    dependencies_rules: BuildFileDependencyRules | None

    @classmethod
    def create(
        cls,
        spec_path: str,
        address_maps: Iterable[AddressMap],
        defaults: BuildFileDefaults = BuildFileDefaults({}),
        dependents_rules: BuildFileDependencyRules | None = None,
        dependencies_rules: BuildFileDependencyRules | None = None,
    ) -> AddressFamily:
        """Creates an address family from the given set of address maps.

        :param spec_path: The directory prefix shared by all address_maps.
        :param address_maps: The family of maps that form this namespace.
        :raises: :class:`MappingError` if the given address maps do not form a family.
        """
        if spec_path == ".":
            spec_path = ""
        for address_map in address_maps:
            if not address_map.path.startswith(spec_path):
                raise DifferingFamiliesError(
                    f"Expected AddressMaps to share the same parent directory {spec_path!r}, "
                    f"but received: {address_map.path!r}"
                )

        name_to_target_adaptors: dict[str, tuple[str, TargetAdaptor]] = {}
        for address_map in address_maps:
            for name, target_adaptor in address_map.name_to_target_adaptor.items():
                if name in name_to_target_adaptors:
                    previous_path, _ = name_to_target_adaptors[name]
                    raise DuplicateNameError(
                        f"A target with name {name!r} is already defined in {previous_path!r}, but "
                        f"is also defined in {address_map.path!r}. Because both targets share the "
                        f"same namespace of {spec_path!r}, this is not allowed."
                    )
                name_to_target_adaptors[name] = (address_map.path, target_adaptor)
        return AddressFamily(
            namespace=spec_path,
            name_to_target_adaptors=dict(sorted(name_to_target_adaptors.items())),
            defaults=defaults,
            dependents_rules=dependents_rules,
            dependencies_rules=dependencies_rules,
        )

    @memoized_property
    def addresses_to_target_adaptors(self) -> Mapping[Address, TargetAdaptor]:
        return {
            Address(spec_path=self.namespace, target_name=name): target_adaptor
            for name, (_, target_adaptor) in self.name_to_target_adaptors.items()
        }

    @memoized_property
    def build_file_addresses(self) -> tuple[BuildFileAddress, ...]:
        return tuple(
            BuildFileAddress(
                rel_path=path, address=Address(spec_path=self.namespace, target_name=name)
            )
            for name, (path, _) in self.name_to_target_adaptors.items()
        )

    @property
    def target_names(self) -> tuple[str, ...]:
        return tuple(addr.target_name for addr in self.addresses_to_target_adaptors)

    def get_target_adaptor(self, address: Address) -> TargetAdaptor | None:
        if self.namespace == DELETED_SPEC_PATH:
            return TargetAdaptor(DELETED_TARGET_TYPE, "deleted_files", "deleted files")
        assert address.spec_path == self.namespace
        entry = self.name_to_target_adaptors.get(address.target_name)
        if entry is None:
            return None
        _, target_adaptor = entry
        return target_adaptor

    def __hash__(self):
        return hash((self.namespace, self.defaults))

    def __repr__(self) -> str:
        return (
            f"AddressFamily(namespace={self.namespace!r}, "
            f"name_to_target_adaptors={sorted(self.name_to_target_adaptors.keys())})"
        )


@dataclass(frozen=True)
class SpecsFilter:
    """Filters targets with the `--tags`, `--exclude-target-regexp`, and `[filter]` subsystem
    options."""

    is_specified: bool
    filter_subsystem_filter: TargetFilter
    tags_filter: TargetFilter

    @classmethod
    def create(
        cls,
        filter_subsystem: FilterSubsystem,
        registered_target_types: RegisteredTargetTypes,
        *,
        tags: Iterable[str],
    ) -> SpecsFilter:
        def tags_outer_filter(tag: str) -> TargetFilter:
            def tags_inner_filter(tgt: Target) -> bool:
                return tag in (tgt.get(Tags).value or [])

            return tags_inner_filter

        tags_filter = and_filters(create_filters(tags, tags_outer_filter))

        return SpecsFilter(
            is_specified=bool(filter_subsystem.is_specified() or tags),
            filter_subsystem_filter=filter_subsystem.all_filters(registered_target_types),
            tags_filter=tags_filter,
        )

    def matches(self, target: Target) -> bool:
        """Check that the target matches the provided `--tag` and `--exclude-target-regexp`
        options."""
        return self.tags_filter(target) and self.filter_subsystem_filter(target)


class AddressFamilies(Collection[AddressFamily]):
    def addresses(self) -> Addresses:
        return Addresses(self._base_addresses())

    def _base_addresses(self) -> Iterable[Address]:
        for family in self:
            yield from family.addresses_to_target_adaptors

1	# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	from __future__ import annotations	12✔
5
6	import os.path	12✔
7	from collections.abc import Iterable, Mapping	12✔
8	from dataclasses import dataclass	12✔
9	from typing import TypeVar	12✔
10
11	from pants.backend.project_info.filter_targets import FilterSubsystem	12✔
12	from pants.base.exceptions import MappingError	12✔
13	from pants.build_graph.address import Address, BuildFileAddress	12✔
14	from pants.engine.addresses import Addresses	12✔
15	from pants.engine.collection import Collection	12✔
16	from pants.engine.env_vars import EnvironmentVars	12✔
17	from pants.engine.internals.defaults import BuildFileDefaults, BuildFileDefaultsParserState	12✔
18	from pants.engine.internals.dep_rules import (	12✔
19	BuildFileDependencyRules,
20	BuildFileDependencyRulesParserState,
21	)
22	from pants.engine.internals.parser import BuildFilePreludeSymbols, Parser	12✔
23	from pants.engine.internals.target_adaptor import TargetAdaptor	12✔
24	from pants.engine.target import RegisteredTargetTypes, Tags, Target	12✔
25	from pants.util.filtering import TargetFilter, and_filters, create_filters	12✔
26	from pants.util.frozendict import FrozenDict	12✔
27	from pants.util.memo import memoized_property	12✔
28
29
30	class DuplicateNameError(MappingError):	12✔
31	"""Indicates more than one top-level object was found with the same name."""
32
33
34	AddressMapT = TypeVar("AddressMapT", bound="AddressMap")	12✔
35
36
37	# A pseudo-spec to refer to deleted paths.
38	DELETED_SPEC_PATH = "__deleted"	12✔
39	# A pseudo-target type to refer to deleted paths.
40	DELETED_TARGET_TYPE = "__deleted_target"	12✔
41
42
43	@dataclass(frozen=True)	12✔
44	class AddressMap:	12✔
45	"""Maps target adaptors from a byte source."""
46
47	path: str	12✔
48	name_to_target_adaptor: FrozenDict[str, TargetAdaptor]	12✔
49
50	@classmethod	12✔
51	def parse(	12✔
52	cls,
53	filepath: str,
54	build_file_content: str,
55	parser: Parser,
56	extra_symbols: BuildFilePreludeSymbols,
57	env_vars: EnvironmentVars,
58	is_bootstrap: bool,
59	defaults: BuildFileDefaultsParserState,
60	dependents_rules: BuildFileDependencyRulesParserState \| None,
61	dependencies_rules: BuildFileDependencyRulesParserState \| None,
62	) -> AddressMap:
63	"""Parses a source for targets.
64
65	The target adaptors are all 'thin': any targets they point to in other namespaces or even in
66	the same namespace but from a separate source are left as unresolved pointers.
67	"""
68	try:	12✔
69	target_adaptors = parser.parse(	12✔
70	filepath,
71	build_file_content,
72	extra_symbols,
73	env_vars,
74	is_bootstrap,
75	defaults,
76	dependents_rules,
77	dependencies_rules,
78	)
79	except Exception as e:	2✔
80	raise MappingError(f"Failed to parse ./{filepath}:\n{type(e).__name__}: {e}")	2✔
81	return cls.create(filepath, target_adaptors)	12✔
82
83	@classmethod	12✔
84	def create(	12✔
85	cls: type[AddressMapT], filepath: str, target_adaptors: Iterable[TargetAdaptor]
86	) -> AddressMapT:
87	name_to_target_adaptors: dict[str, TargetAdaptor] = {}	12✔
88	for target_adaptor in target_adaptors:	12✔
89	name = target_adaptor.name or os.path.basename(os.path.dirname(filepath))	12✔
90	if name in name_to_target_adaptors:	12✔
91	duplicate = name_to_target_adaptors[name]	1✔
92	raise DuplicateNameError(	1✔
93	f"A target already exists at {filepath!r} with name {name!r} and target type "
94	f"{duplicate.type_alias!r}. The {target_adaptor.type_alias!r} target "
95	"cannot use the same name."
96	)
97	name_to_target_adaptors[name] = target_adaptor	12✔
98	return cls(filepath, FrozenDict(sorted(name_to_target_adaptors.items())))	12✔
99
100
101	class DifferingFamiliesError(MappingError):	12✔
102	"""Indicates an attempt was made to merge address maps from different families together."""
103
104
105	@dataclass(frozen=True)	12✔
106	class AddressFamily:	12✔
107	"""Represents the family of target adaptors collected from the BUILD files in one directory.
108
109	To create an AddressFamily, use `create`.
110
111	An address family can be composed of the target adaptors from zero or more underlying address
112	sources. An "empty" AddressFamily is legal, and is the result when there are not build files in
113	a particular namespace.
114
115	:param namespace: The namespace path of this address family.
116	:param name_to_target_adaptors: A dict mapping from name to the target adaptor.
117	:param defaults: The default target field values, per target type, applicable for this address family.
118	:param dependents_rules: The rules to apply on incoming dependencies to targets in this family.
119	:param dependencies_rules: The rules to apply on the outgoing dependencies from targets in this family.
120	"""
121
122	# The directory from which the adaptors were parsed.
123	namespace: str	12✔
124	name_to_target_adaptors: dict[str, tuple[str, TargetAdaptor]]	12✔
125	defaults: BuildFileDefaults	12✔
126	dependents_rules: BuildFileDependencyRules \| None	12✔
127	dependencies_rules: BuildFileDependencyRules \| None	12✔
128
129	@classmethod	12✔
130	def create(	12✔
131	cls,
132	spec_path: str,
133	address_maps: Iterable[AddressMap],
134	defaults: BuildFileDefaults = BuildFileDefaults({}),
135	dependents_rules: BuildFileDependencyRules \| None = None,
136	dependencies_rules: BuildFileDependencyRules \| None = None,
137	) -> AddressFamily:
138	"""Creates an address family from the given set of address maps.
139
140	:param spec_path: The directory prefix shared by all address_maps.
141	:param address_maps: The family of maps that form this namespace.
142	:raises: :class:`MappingError` if the given address maps do not form a family.
143	"""
144	if spec_path == ".":	12✔
145	spec_path = ""	12✔
146	for address_map in address_maps:	12✔
147	if not address_map.path.startswith(spec_path):	12✔
148	raise DifferingFamiliesError(	1✔
149	f"Expected AddressMaps to share the same parent directory {spec_path!r}, "
150	f"but received: {address_map.path!r}"
151	)
152
153	name_to_target_adaptors: dict[str, tuple[str, TargetAdaptor]] = {}	12✔
154	for address_map in address_maps:	12✔
155	for name, target_adaptor in address_map.name_to_target_adaptor.items():	12✔
156	if name in name_to_target_adaptors:	12✔
157	previous_path, _ = name_to_target_adaptors[name]	1✔
158	raise DuplicateNameError(	1✔
159	f"A target with name {name!r} is already defined in {previous_path!r}, but "
160	f"is also defined in {address_map.path!r}. Because both targets share the "
161	f"same namespace of {spec_path!r}, this is not allowed."
162	)
163	name_to_target_adaptors[name] = (address_map.path, target_adaptor)	12✔
164	return AddressFamily(	12✔
165	namespace=spec_path,
166	name_to_target_adaptors=dict(sorted(name_to_target_adaptors.items())),
167	defaults=defaults,
168	dependents_rules=dependents_rules,
169	dependencies_rules=dependencies_rules,
170	)
171
172	@memoized_property	12✔
173	def addresses_to_target_adaptors(self) -> Mapping[Address, TargetAdaptor]:	12✔
174	return {	12✔
175	Address(spec_path=self.namespace, target_name=name): target_adaptor
176	for name, (_, target_adaptor) in self.name_to_target_adaptors.items()
177	}
178
179	@memoized_property	12✔
180	def build_file_addresses(self) -> tuple[BuildFileAddress, ...]:	12✔
181	return tuple(	11✔
182	BuildFileAddress(
183	rel_path=path, address=Address(spec_path=self.namespace, target_name=name)
184	)
185	for name, (path, _) in self.name_to_target_adaptors.items()
186	)
187
188	@property	12✔
189	def target_names(self) -> tuple[str, ...]:	12✔
190	return tuple(addr.target_name for addr in self.addresses_to_target_adaptors)	5✔
191
192	def get_target_adaptor(self, address: Address) -> TargetAdaptor \| None:	12✔
193	if self.namespace == DELETED_SPEC_PATH:	12✔
NEW 194	return TargetAdaptor(DELETED_TARGET_TYPE, "deleted_files", "deleted files")	×
195	assert address.spec_path == self.namespace	12✔
196	entry = self.name_to_target_adaptors.get(address.target_name)	12✔
197	if entry is None:	12✔
198	return None	5✔
199	_, target_adaptor = entry	12✔
200	return target_adaptor	12✔
201
202	def __hash__(self):	12✔
203	return hash((self.namespace, self.defaults))	×
204
205	def __repr__(self) -> str:	12✔
206	return (	×
207	f"AddressFamily(namespace={self.namespace!r}, "
208	f"name_to_target_adaptors={sorted(self.name_to_target_adaptors.keys())})"
209	)
210
211
212	@dataclass(frozen=True)	12✔
213	class SpecsFilter:	12✔
214	"""Filters targets with the `--tags`, `--exclude-target-regexp`, and `[filter]` subsystem
215	options."""
216
217	is_specified: bool	12✔
218	filter_subsystem_filter: TargetFilter	12✔
219	tags_filter: TargetFilter	12✔
220
221	@classmethod	12✔
222	def create(	12✔
223	cls,
224	filter_subsystem: FilterSubsystem,
225	registered_target_types: RegisteredTargetTypes,
226	*,
227	tags: Iterable[str],
228	) -> SpecsFilter:
229	def tags_outer_filter(tag: str) -> TargetFilter:	12✔
230	def tags_inner_filter(tgt: Target) -> bool:	2✔
231	return tag in (tgt.get(Tags).value or [])	2✔
232
233	return tags_inner_filter	2✔
234
235	tags_filter = and_filters(create_filters(tags, tags_outer_filter))	12✔
236
237	return SpecsFilter(	12✔
238	is_specified=bool(filter_subsystem.is_specified() or tags),
239	filter_subsystem_filter=filter_subsystem.all_filters(registered_target_types),
240	tags_filter=tags_filter,
241	)
242
243	def matches(self, target: Target) -> bool:	12✔
244	"""Check that the target matches the provided `--tag` and `--exclude-target-regexp`
245	options."""
246	return self.tags_filter(target) and self.filter_subsystem_filter(target)	12✔
247
248
249	class AddressFamilies(Collection[AddressFamily]):	12✔
250	def addresses(self) -> Addresses:	12✔
251	return Addresses(self._base_addresses())	12✔
252
253	def _base_addresses(self) -> Iterable[Address]:	12✔
254	for family in self:	12✔
255	yield from family.addresses_to_target_adaptors	12✔

pantsbuild / pants / 25357472185

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous