25357472185

Committed 05 May 2026 04:13AM UTC coverage: 92.941% (-0.02%) from 92.956%

Build # 25357472185

Build Type

push

github

Committed by

web-flow

Commit Message

Better change detection on deleted files (#23311)

There are several longstanding issues on this:

https://github.com/pantsbuild/pants/issues/13232
https://github.com/pantsbuild/pants/issues/14975
https://github.com/pantsbuild/pants/issues/17512
https://github.com/pantsbuild/pants/issues/23240

This has historically been intractable because the
design of dep inference and of change detection
are at odds with each other.

A fully robust solution is still somewhat out of reach,
but this change provides a general-purpose mechanism
that backends can opt in to, and uses that mechanism
in the python backend.

The result is a substantial yet incremental improvement
in the most prominent use case.

The limitation of this mechanism is that it can only
be used if a backend can infer a likely dependency on
a file simply from its path. In Python this is possible because
dotted module paths must recapitulate the filesystem path
from the source root. But this is not true in general for
all languages.

Coverage Stats

56 of 78 new or added lines in 8 files covered. (71.79%)

4 existing lines in 3 files now uncovered.

91907 of 98887 relevant lines covered (92.94%)

4.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

43.86

/src/python/pants/init/specs_calculator.py

# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import logging
from typing import cast

from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
from pants.base.specs import AddressLiteralSpec, FileLiteralSpec, RawSpecs, Specs
from pants.base.specs_parser import SpecsParser
from pants.core.environments.rules import determine_bootstrap_environment
from pants.core.util_rules.system_binaries import GitBinary
from pants.engine.addresses import AddressInput
from pants.engine.environment import EnvironmentName
from pants.engine.internals.build_files import DELETED_ADDRESS
from pants.engine.internals.graph import FilesWithSourceBlocks
from pants.engine.internals.scheduler import SchedulerSession
from pants.engine.internals.selectors import Params
from pants.engine.rules import QueryRule
from pants.option.options import Options
from pants.option.options_bootstrapper import OptionsBootstrapper
from pants.util.frozendict import FrozenDict
from pants.vcs.changed import ChangedAddresses, ChangedOptions, ChangedRequest
from pants.vcs.git import GitWorktreeRequest, MaybeGitWorktree
from pants.vcs.hunk import TextBlocks

logger = logging.getLogger(__name__)


class InvalidSpecConstraint(Exception):
    """Raised when invalid constraints are given via specs and arguments like --changed*."""


def calculate_specs(
    options_bootstrapper: OptionsBootstrapper,
    options: Options,
    session: SchedulerSession,
    working_dir: str,
) -> Specs:
    """Determine the specs for a given Pants run."""
    global_options = options.for_global_scope()
    unmatched_cli_globs = global_options.unmatched_cli_globs
    specs = SpecsParser(working_dir=working_dir).parse_specs(
        options.specs,
        description_of_origin="CLI arguments",
        unmatched_glob_behavior=unmatched_cli_globs,
    )

    changed_options = ChangedOptions.from_options(options.for_scope("changed"))
    logger.debug("specs are: %s", specs)
    logger.debug("changed_options are: %s", changed_options)

    if specs and changed_options.provided:
        changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"
        specs_description = specs.arguments_provided_description()
        assert specs_description is not None
        raise InvalidSpecConstraint(
            f"You used `{changed_name}` at the same time as using {specs_description}. You can "
            f"only use `{changed_name}` or use normal arguments."
        )

    if not changed_options.provided:
        return specs

    bootstrap_environment = determine_bootstrap_environment(session)

    (git_binary,) = session.product_request(GitBinary, Params(bootstrap_environment))
    (maybe_git_worktree,) = session.product_request(
        MaybeGitWorktree, Params(GitWorktreeRequest(), git_binary, bootstrap_environment)
    )
    if not maybe_git_worktree.git_worktree:
        raise InvalidSpecConstraint(
            "The `--changed-*` options are only available if Git is used for the repository."
        )

    (files_with_sources_blocks,) = session.product_request(
        FilesWithSourceBlocks, Params(bootstrap_environment)
    )
    changed_files = tuple(
        file
        for file in changed_options.changed_files(maybe_git_worktree.git_worktree)
        # We want to exclude the file from the normal processing flow if it has associated
        # targets with text blocks. These files are handled with special logic.
        if file not in files_with_sources_blocks
    )
    file_literal_specs = tuple(FileLiteralSpec(f) for f in changed_files)

    sources_blocks = FrozenDict(
        (
            path,
            # Hunk stores information about the old block and the new block.
            # Here we only care about the final state, so we take `hunk.right`.
            TextBlocks(hunk.right for hunk in hunks if hunk.right is not None),
        )
        for path, hunks in changed_options.diff_hunks(
            maybe_git_worktree.git_worktree,
            files_with_sources_blocks,
        ).items()
    )
    logger.debug("changed text blocks: %s", sources_blocks)

    changed_request = ChangedRequest(
        sources=changed_files,
        dependents=changed_options.dependents,
        sources_blocks=sources_blocks,
    )
    (changed_addresses,) = session.product_request(
        ChangedAddresses,
        Params(changed_request, options_bootstrapper, bootstrap_environment),
    )
    logger.debug("changed addresses: %s", changed_addresses)

    address_literal_specs = []
    for address in cast(ChangedAddresses, changed_addresses):
        address_input = AddressInput.parse(address.spec, description_of_origin="`--changed-since`")
        # The pseudo-target has done its job, but we don't want to actually see it downstream.
        if address != DELETED_ADDRESS:
            address_literal_specs.append(
                AddressLiteralSpec(
                    path_component=address_input.path_component,
                    target_component=address_input.target_component,
                    generated_component=address_input.generated_component,
                    parameters=FrozenDict(address_input.parameters),
                )
            )

    return Specs(
        includes=RawSpecs(
            # We need both address_literals and file_literals to cover all our edge cases, including
            # target-aware vs. target-less goals, e.g. `list` vs `count-loc`.
            address_literals=tuple(address_literal_specs),
            file_literals=file_literal_specs,
            # The globs here are synthesized from VCS data by the `changed` mechanism.
            # As such it does not make sense to apply user-facing matching errors to them.
            # In particular, they can legitimately not match anything, if entire git
            # subtrees were deleted for example.
            unmatched_glob_behavior=GlobMatchErrorBehavior.ignore,
            filter_by_global_options=True,
            from_change_detection=True,
            description_of_origin="`--changed-since`",
        ),
        ignores=RawSpecs(description_of_origin="`--changed-since`"),
    )


def rules():
    return [
        QueryRule(ChangedAddresses, [ChangedRequest, EnvironmentName]),
        QueryRule(GitBinary, [EnvironmentName]),
        QueryRule(MaybeGitWorktree, [GitWorktreeRequest, GitBinary, EnvironmentName]),
        QueryRule(FilesWithSourceBlocks, [EnvironmentName]),
    ]

1	# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	import logging	12✔
5	from typing import cast	12✔
6
7	from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior	12✔
8	from pants.base.specs import AddressLiteralSpec, FileLiteralSpec, RawSpecs, Specs	12✔
9	from pants.base.specs_parser import SpecsParser	12✔
10	from pants.core.environments.rules import determine_bootstrap_environment	12✔
11	from pants.core.util_rules.system_binaries import GitBinary	12✔
12	from pants.engine.addresses import AddressInput	12✔
13	from pants.engine.environment import EnvironmentName	12✔
14	from pants.engine.internals.build_files import DELETED_ADDRESS	12✔
15	from pants.engine.internals.graph import FilesWithSourceBlocks	12✔
16	from pants.engine.internals.scheduler import SchedulerSession	12✔
17	from pants.engine.internals.selectors import Params	12✔
18	from pants.engine.rules import QueryRule	12✔
19	from pants.option.options import Options	12✔
20	from pants.option.options_bootstrapper import OptionsBootstrapper	12✔
21	from pants.util.frozendict import FrozenDict	12✔
22	from pants.vcs.changed import ChangedAddresses, ChangedOptions, ChangedRequest	12✔
23	from pants.vcs.git import GitWorktreeRequest, MaybeGitWorktree	12✔
24	from pants.vcs.hunk import TextBlocks	12✔
25
26	logger = logging.getLogger(__name__)	12✔
27
28
29	class InvalidSpecConstraint(Exception):	12✔
30	"""Raised when invalid constraints are given via specs and arguments like --changed*."""
31
32
33	def calculate_specs(	12✔
34	options_bootstrapper: OptionsBootstrapper,
35	options: Options,
36	session: SchedulerSession,
37	working_dir: str,
38	) -> Specs:
39	"""Determine the specs for a given Pants run."""
40	global_options = options.for_global_scope()	×
41	unmatched_cli_globs = global_options.unmatched_cli_globs	×
42	specs = SpecsParser(working_dir=working_dir).parse_specs(	×
43	options.specs,
44	description_of_origin="CLI arguments",
45	unmatched_glob_behavior=unmatched_cli_globs,
46	)
47
48	changed_options = ChangedOptions.from_options(options.for_scope("changed"))	×
49	logger.debug("specs are: %s", specs)	×
50	logger.debug("changed_options are: %s", changed_options)	×
51
52	if specs and changed_options.provided:	×
53	changed_name = "--changed-since" if changed_options.since else "--changed-diffspec"	×
54	specs_description = specs.arguments_provided_description()	×
55	assert specs_description is not None	×
56	raise InvalidSpecConstraint(	×
57	f"You used `{changed_name}` at the same time as using {specs_description}. You can "
58	f"only use `{changed_name}` or use normal arguments."
59	)
60
61	if not changed_options.provided:	×
62	return specs	×
63
64	bootstrap_environment = determine_bootstrap_environment(session)	×
65
66	(git_binary,) = session.product_request(GitBinary, Params(bootstrap_environment))	×
67	(maybe_git_worktree,) = session.product_request(	×
68	MaybeGitWorktree, Params(GitWorktreeRequest(), git_binary, bootstrap_environment)
69	)
70	if not maybe_git_worktree.git_worktree:	×
71	raise InvalidSpecConstraint(	×
72	"The `--changed-*` options are only available if Git is used for the repository."
73	)
74
75	(files_with_sources_blocks,) = session.product_request(	×
76	FilesWithSourceBlocks, Params(bootstrap_environment)
77	)
78	changed_files = tuple(	×
79	file
80	for file in changed_options.changed_files(maybe_git_worktree.git_worktree)
81	# We want to exclude the file from the normal processing flow if it has associated
82	# targets with text blocks. These files are handled with special logic.
83	if file not in files_with_sources_blocks
84	)
85	file_literal_specs = tuple(FileLiteralSpec(f) for f in changed_files)	×
86
87	sources_blocks = FrozenDict(	×
88	(
89	path,
90	# Hunk stores information about the old block and the new block.
91	# Here we only care about the final state, so we take `hunk.right`.
92	TextBlocks(hunk.right for hunk in hunks if hunk.right is not None),
93	)
94	for path, hunks in changed_options.diff_hunks(
95	maybe_git_worktree.git_worktree,
96	files_with_sources_blocks,
97	).items()
98	)
99	logger.debug("changed text blocks: %s", sources_blocks)	×
100
101	changed_request = ChangedRequest(	×
102	sources=changed_files,
103	dependents=changed_options.dependents,
104	sources_blocks=sources_blocks,
105	)
106	(changed_addresses,) = session.product_request(	×
107	ChangedAddresses,
108	Params(changed_request, options_bootstrapper, bootstrap_environment),
109	)
110	logger.debug("changed addresses: %s", changed_addresses)	×
111
112	address_literal_specs = []	×
113	for address in cast(ChangedAddresses, changed_addresses):	×
114	address_input = AddressInput.parse(address.spec, description_of_origin="`--changed-since`")	×
115	# The pseudo-target has done its job, but we don't want to actually see it downstream.
NEW 116	if address != DELETED_ADDRESS:	×
NEW 117	address_literal_specs.append(	×
118	AddressLiteralSpec(
119	path_component=address_input.path_component,
120	target_component=address_input.target_component,
121	generated_component=address_input.generated_component,
122	parameters=FrozenDict(address_input.parameters),
123	)
124	)
125
UNCOV 126	return Specs(	×
127	includes=RawSpecs(
128	# We need both address_literals and file_literals to cover all our edge cases, including
129	# target-aware vs. target-less goals, e.g. `list` vs `count-loc`.
130	address_literals=tuple(address_literal_specs),
131	file_literals=file_literal_specs,
132	# The globs here are synthesized from VCS data by the `changed` mechanism.
133	# As such it does not make sense to apply user-facing matching errors to them.
134	# In particular, they can legitimately not match anything, if entire git
135	# subtrees were deleted for example.
136	unmatched_glob_behavior=GlobMatchErrorBehavior.ignore,
137	filter_by_global_options=True,
138	from_change_detection=True,
139	description_of_origin="`--changed-since`",
140	),
141	ignores=RawSpecs(description_of_origin="`--changed-since`"),
142	)
143
144
145	def rules():	12✔
146	return [	12✔
147	QueryRule(ChangedAddresses, [ChangedRequest, EnvironmentName]),
148	QueryRule(GitBinary, [EnvironmentName]),
149	QueryRule(MaybeGitWorktree, [GitWorktreeRequest, GitBinary, EnvironmentName]),
150	QueryRule(FilesWithSourceBlocks, [EnvironmentName]),
151	]

pantsbuild / pants / 25357472185

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous