24103052497

Committed 07 Apr 2026 08:33PM UTC coverage: 52.311% (-40.6%) from 92.909%

Build # 24103052497

Build Type

Pull #23228

github

Committed by

web-flow

Commit Message

Merge 18fdfb0fd into b05152cd9

Pull Request Pull Request #23228: Add persistent dependency inference cache for incremental --changed-dependents

Coverage Stats

31 of 136 new or added lines in 2 files covered. (22.79%)

23028 existing lines in 605 files now uncovered.

31671 of 60544 relevant lines covered (52.31%)

1.05 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

30.12

/src/python/pants/backend/project_info/incremental_dependents.py

# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

"""Incremental dependency graph updates for faster `--changed-dependents` runs.

Instead of resolving dependencies for ALL targets every time, this module persists
the forward dependency graph to disk and only re-resolves dependencies for targets
whose source files have changed since the last run.
"""

from __future__ import annotations

import hashlib
import json
import logging
import os
from dataclasses import dataclass
from pants.base.build_environment import get_pants_cachedir
from pants.option.option_types import BoolOption
from pants.option.subsystem import Subsystem
from pants.util.strutil import help_text

logger = logging.getLogger(__name__)


class IncrementalDependents(Subsystem):
    options_scope = "incremental-dependents"
    help = help_text(
        """
        Persist the forward dependency graph to disk and incrementally update it,
        so that `--changed-dependents=transitive` does not need to resolve
        dependencies for every target on every run.
        """
    )

    enabled = BoolOption(
        default=False,
        help="Enable incremental dependency graph caching. "
        "When enabled, the forward dependency graph is persisted to disk and only "
        "targets with changed source files have their dependencies re-resolved.",
    )


# ---------------------------------------------------------------------------
# Persisted graph helpers
# ---------------------------------------------------------------------------

_CACHE_VERSION = 2  # v2: stores structured address components


@dataclass(frozen=True)
class CachedEntry:
    fingerprint: str
    # Dependencies stored as address spec strings (e.g. "src/python/foo/bar.py:lib")
    deps: tuple[str, ...]


def get_cache_path() -> str:
    """Return the path to the incremental dep graph cache file."""
    return os.path.join(get_pants_cachedir(), "incremental_dep_graph_v2.json")


def load_persisted_graph(path: str, buildroot: str) -> dict[str, CachedEntry]:
    """Load the persisted forward dependency graph from disk.

    Returns an empty dict if the file doesn't exist or is invalid.
    """
    try:
        with open(path) as f:
            data = json.load(f)
        if data.get("version") != _CACHE_VERSION:
            logger.debug("Incremental dep graph cache version mismatch, rebuilding.")
            return {}
        if data.get("buildroot") != buildroot:
            logger.debug("Incremental dep graph cache buildroot mismatch, rebuilding.")
            return {}
        entries: dict[str, CachedEntry] = {}
        for addr_spec, entry in data.get("entries", {}).items():
            entries[addr_spec] = CachedEntry(
                fingerprint=entry["fingerprint"],
                deps=tuple(entry["deps"]),
            )
        return entries
    except (FileNotFoundError, json.JSONDecodeError, KeyError, TypeError) as e:
        logger.debug("Could not load incremental dep graph cache: %s", e)
        return {}


def save_persisted_graph(
    path: str,
    buildroot: str,
    entries: dict[str, CachedEntry],
) -> None:
    """Save the forward dependency graph to disk."""
    data = {
        "version": _CACHE_VERSION,
        "buildroot": buildroot,
        "entries": {
            addr_spec: {
                "fingerprint": entry.fingerprint,
                "deps": list(entry.deps),
            }
            for addr_spec, entry in entries.items()
        },
    }
    os.makedirs(os.path.dirname(path), exist_ok=True)

    # Atomic write: write to temp file then rename
    tmp_path = path + ".tmp"
    try:
        with open(tmp_path, "w") as f:
            json.dump(data, f, separators=(",", ":"))
        os.replace(tmp_path, path)
        logger.debug(
            "Saved incremental dep graph cache with %d entries to %s",
            len(entries),
            path,
        )
    except OSError as e:
        logger.warning("Failed to save incremental dep graph cache: %s", e)
        try:
            os.unlink(tmp_path)
        except OSError:
            pass


def _sha256_file(path: str) -> str | None:
    """Return the SHA-256 hex digest of a file's contents, or None if unreadable."""
    try:
        h = hashlib.sha256()
        with open(path, "rb") as f:
            for chunk in iter(lambda: f.read(65536), b""):
                h.update(chunk)
        return h.hexdigest()
    except OSError:
        return None


def compute_source_fingerprint(target_address: Address, buildroot: str) -> str:
    """Compute a content-based fingerprint for a target.

    Uses SHA-256 of file contents (not mtime) so the cache is portable across
    machines — critical for CI where git clone sets all mtimes to the same value.

    The fingerprint includes:
    - The BUILD file defining the target
    - The specific source file (for generated/file-level targets)
    """
    hasher = hashlib.sha256()

    # Always include the BUILD file(s) in the fingerprint
    spec_path = target_address.spec_path
    build_dir = os.path.join(buildroot, spec_path) if spec_path else buildroot

    for build_name in ("BUILD", "BUILD.pants"):
        build_file = os.path.join(build_dir, build_name)
        digest = _sha256_file(build_file)
        if digest:
            hasher.update(f"BUILD:{build_file}:{digest}".encode())

    # For file-addressed targets (e.g. python_source generated from python_sources),
    # include the file's own content hash.
    if target_address.is_generated_target and target_address.generated_name:
        gen_name = target_address.generated_name
        candidate = (
            os.path.join(buildroot, spec_path, gen_name)
            if spec_path
            else os.path.join(buildroot, gen_name)
        )
        digest = _sha256_file(candidate)
        if digest:
            hasher.update(f"SRC:{candidate}:{digest}".encode())
        elif candidate != os.path.join(buildroot, gen_name):
            # Also try as a path directly from buildroot
            digest = _sha256_file(os.path.join(buildroot, gen_name))
            if digest:
                hasher.update(f"SRC:{gen_name}:{digest}".encode())

    return hasher.hexdigest()

1	# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	"""Incremental dependency graph updates for faster `--changed-dependents` runs.
5
6	Instead of resolving dependencies for ALL targets every time, this module persists
7	the forward dependency graph to disk and only re-resolves dependencies for targets
8	whose source files have changed since the last run.
9	"""
10
11	from __future__ import annotations	2✔
12
13	import hashlib	2✔
14	import json	2✔
15	import logging	2✔
16	import os	2✔
17	from dataclasses import dataclass	2✔
18	from pants.base.build_environment import get_pants_cachedir	2✔
19	from pants.option.option_types import BoolOption	2✔
20	from pants.option.subsystem import Subsystem	2✔
21	from pants.util.strutil import help_text	2✔
22
23	logger = logging.getLogger(__name__)	2✔
24
25
26	class IncrementalDependents(Subsystem):	2✔
27	options_scope = "incremental-dependents"	2✔
28	help = help_text(	2✔
29	"""
30	Persist the forward dependency graph to disk and incrementally update it,
31	so that `--changed-dependents=transitive` does not need to resolve
32	dependencies for every target on every run.
33	"""
34	)
35
36	enabled = BoolOption(	2✔
37	default=False,
38	help="Enable incremental dependency graph caching. "
39	"When enabled, the forward dependency graph is persisted to disk and only "
40	"targets with changed source files have their dependencies re-resolved.",
41	)
42
43
44	# ---------------------------------------------------------------------------
45	# Persisted graph helpers
46	# ---------------------------------------------------------------------------
47
48	_CACHE_VERSION = 2 # v2: stores structured address components	2✔
49
50
51	@dataclass(frozen=True)	2✔
52	class CachedEntry:	2✔
53	fingerprint: str	2✔
54	# Dependencies stored as address spec strings (e.g. "src/python/foo/bar.py:lib")
55	deps: tuple[str, ...]	2✔
56
57
58	def get_cache_path() -> str:	2✔
59	"""Return the path to the incremental dep graph cache file."""
NEW 60	return os.path.join(get_pants_cachedir(), "incremental_dep_graph_v2.json")	×
61
62
63	def load_persisted_graph(path: str, buildroot: str) -> dict[str, CachedEntry]:	2✔
64	"""Load the persisted forward dependency graph from disk.
65
66	Returns an empty dict if the file doesn't exist or is invalid.
67	"""
NEW 68	try:	×
NEW 69	with open(path) as f:	×
NEW 70	data = json.load(f)	×
NEW 71	if data.get("version") != _CACHE_VERSION:	×
NEW 72	logger.debug("Incremental dep graph cache version mismatch, rebuilding.")	×
NEW 73	return {}	×
NEW 74	if data.get("buildroot") != buildroot:	×
NEW 75	logger.debug("Incremental dep graph cache buildroot mismatch, rebuilding.")	×
NEW 76	return {}	×
NEW 77	entries: dict[str, CachedEntry] = {}	×
NEW 78	for addr_spec, entry in data.get("entries", {}).items():	×
NEW 79	entries[addr_spec] = CachedEntry(	×
80	fingerprint=entry["fingerprint"],
81	deps=tuple(entry["deps"]),
82	)
NEW 83	return entries	×
NEW 84	except (FileNotFoundError, json.JSONDecodeError, KeyError, TypeError) as e:	×
NEW 85	logger.debug("Could not load incremental dep graph cache: %s", e)	×
NEW 86	return {}	×
87
88
89	def save_persisted_graph(	2✔
90	path: str,
91	buildroot: str,
92	entries: dict[str, CachedEntry],
93	) -> None:
94	"""Save the forward dependency graph to disk."""
NEW 95	data = {	×
96	"version": _CACHE_VERSION,
97	"buildroot": buildroot,
98	"entries": {
99	addr_spec: {
100	"fingerprint": entry.fingerprint,
101	"deps": list(entry.deps),
102	}
103	for addr_spec, entry in entries.items()
104	},
105	}
NEW 106	os.makedirs(os.path.dirname(path), exist_ok=True)	×
107
108	# Atomic write: write to temp file then rename
NEW 109	tmp_path = path + ".tmp"	×
NEW 110	try:	×
NEW 111	with open(tmp_path, "w") as f:	×
NEW 112	json.dump(data, f, separators=(",", ":"))	×
NEW 113	os.replace(tmp_path, path)	×
NEW 114	logger.debug(	×
115	"Saved incremental dep graph cache with %d entries to %s",
116	len(entries),
117	path,
118	)
NEW 119	except OSError as e:	×
NEW 120	logger.warning("Failed to save incremental dep graph cache: %s", e)	×
NEW 121	try:	×
NEW 122	os.unlink(tmp_path)	×
NEW 123	except OSError:	×
NEW 124	pass	×
125
126
127	def _sha256_file(path: str) -> str \| None:	2✔
128	"""Return the SHA-256 hex digest of a file's contents, or None if unreadable."""
NEW 129	try:	×
NEW 130	h = hashlib.sha256()	×
NEW 131	with open(path, "rb") as f:	×
NEW 132	for chunk in iter(lambda: f.read(65536), b""):	×
NEW 133	h.update(chunk)	×
NEW 134	return h.hexdigest()	×
NEW 135	except OSError:	×
NEW 136	return None	×
137
138
139	def compute_source_fingerprint(target_address: Address, buildroot: str) -> str:	2✔
140	"""Compute a content-based fingerprint for a target.
141
142	Uses SHA-256 of file contents (not mtime) so the cache is portable across
143	machines — critical for CI where git clone sets all mtimes to the same value.
144
145	The fingerprint includes:
146	- The BUILD file defining the target
147	- The specific source file (for generated/file-level targets)
148	"""
NEW 149	hasher = hashlib.sha256()	×
150
151	# Always include the BUILD file(s) in the fingerprint
NEW 152	spec_path = target_address.spec_path	×
NEW 153	build_dir = os.path.join(buildroot, spec_path) if spec_path else buildroot	×
154
NEW 155	for build_name in ("BUILD", "BUILD.pants"):	×
NEW 156	build_file = os.path.join(build_dir, build_name)	×
NEW 157	digest = _sha256_file(build_file)	×
NEW 158	if digest:	×
NEW 159	hasher.update(f"BUILD:{build_file}:{digest}".encode())	×
160
161	# For file-addressed targets (e.g. python_source generated from python_sources),
162	# include the file's own content hash.
NEW 163	if target_address.is_generated_target and target_address.generated_name:	×
NEW 164	gen_name = target_address.generated_name	×
NEW 165	candidate = (	×
166	os.path.join(buildroot, spec_path, gen_name)
167	if spec_path
168	else os.path.join(buildroot, gen_name)
169	)
NEW 170	digest = _sha256_file(candidate)	×
NEW 171	if digest:	×
NEW 172	hasher.update(f"SRC:{candidate}:{digest}".encode())	×
NEW 173	elif candidate != os.path.join(buildroot, gen_name):	×
174	# Also try as a path directly from buildroot
NEW 175	digest = _sha256_file(os.path.join(buildroot, gen_name))	×
NEW 176	if digest:	×
NEW 177	hasher.update(f"SRC:{gen_name}:{digest}".encode())	×
178
NEW 179	return hasher.hexdigest()	×

pantsbuild / pants / 24103052497

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous