21726110101

Committed 05 Feb 2026 07:49PM UTC coverage: 80.284% (-0.01%) from 80.296%

Build # 21726110101

Build Type

push

github

Committed by

web-flow

Commit Message

Support dep inference on multiple files in a single workunit (#23075)

Each file's result is still cached independently.

In large repos we schedule a large number of very fast
tasks, so the scheduling overhead dominates the runtime,
so processing many files in a batch is a win.

The Python-side code doesn't make use of this yet, so it
passes single files and receives singletons back.

A followup change can actually put this into action.

Run Details

20 of 44 new or added lines in 5 files covered. (45.45%)

2 existing lines in 2 files now uncovered.

78530 of 97815 relevant lines covered (80.28%)

3.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.2

/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py

# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
from __future__ import annotations

import logging
import os
from collections.abc import Iterable
from dataclasses import dataclass

from pants.backend.python.dependency_inference.subsystem import PythonInferSubsystem
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.core.util_rules.source_files import SourceFiles
from pants.core.util_rules.stripped_source_files import strip_source_roots
from pants.engine.collection import DeduplicatedCollection
from pants.engine.fs import CreateDigest, Digest, FileContent
from pants.engine.internals.native_engine import NativeDependenciesRequest
from pants.engine.intrinsics import create_digest, parse_python_deps
from pants.engine.rules import collect_rules, rule
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel
from pants.util.resources import read_resource

logger = logging.getLogger(__name__)


@dataclass(frozen=True, order=True)
class ParsedPythonImportInfo:
    lineno: int
    # An import is considered "weak" if we're unsure if a dependency will exist between the parsed
    # file and the parsed import.
    # Examples of "weak" imports include string imports (if enabled) or those inside a try block
    # which has a handler catching ImportError.
    weak: bool


class ParsedPythonImports(FrozenDict[str, ParsedPythonImportInfo]):
    """All the discovered imports from a Python source file mapped to the relevant info."""


class ParsedPythonAssetPaths(DeduplicatedCollection[str]):
    """All the discovered possible assets from a Python source file."""

    # N.B. Don't set `sort_input`, as the input is already sorted


@dataclass(frozen=True)
class ParsedPythonDependencies:
    imports: ParsedPythonImports
    assets: ParsedPythonAssetPaths


@dataclass(frozen=True)
class ParsePythonDependenciesRequest:
    source: SourceFiles
    interpreter_constraints: InterpreterConstraints


@dataclass(frozen=True)
class PythonDependencyVisitor:
    """Wraps a subclass of DependencyVisitorBase."""

    digest: Digest  # The file contents for the visitor
    classname: str  # The full classname, e.g., _my_custom_dep_parser.MyCustomVisitor
    env: FrozenDict[str, str]  # Set these env vars when invoking the visitor


@dataclass(frozen=True)
class ParserScript:
    digest: Digest
    env: FrozenDict[str, str]


_scripts_package = "pants.backend.python.dependency_inference.scripts"


async def get_scripts_digest(scripts_package: str, filenames: Iterable[str]) -> Digest:
    scripts = [read_resource(scripts_package, filename) for filename in filenames]
    assert all(script is not None for script in scripts)
    path_prefix = scripts_package.replace(".", os.path.sep)
    contents = [
        FileContent(os.path.join(path_prefix, relpath), script)
        for relpath, script in zip(filenames, scripts)
    ]

    # Python 2 requires all the intermediate __init__.py to exist in the sandbox.
    package = scripts_package
    while package:
        contents.append(
            FileContent(
                os.path.join(package.replace(".", os.path.sep), "__init__.py"),
                read_resource(package, "__init__.py"),
            )
        )
        package = package.rpartition(".")[0]

    digest = await create_digest(CreateDigest(contents))
    return digest


@rule(level=LogLevel.DEBUG)
async def parse_python_dependencies(
    request: ParsePythonDependenciesRequest,
    python_infer_subsystem: PythonInferSubsystem,
) -> ParsedPythonDependencies:
    stripped_sources = await strip_source_roots(request.source)
    # We operate on PythonSourceField, which should be one file.
    assert len(stripped_sources.snapshot.files) == 1

    native_results = await parse_python_deps(
        NativeDependenciesRequest(stripped_sources.snapshot.digest)
    )
    assert len(native_results.path_to_deps) == 1
    native_result = next(iter(native_results.path_to_deps.values()))
    imports = dict(native_result.imports)
    assets = set()

    if python_infer_subsystem.string_imports or python_infer_subsystem.assets:
        for string, line in native_result.string_candidates.items():
            if (
                python_infer_subsystem.string_imports
                and string.count(".") >= python_infer_subsystem.string_imports_min_dots
                and all(part.isidentifier() for part in string.split("."))
            ):
                imports.setdefault(string, (line, True))
            if (
                python_infer_subsystem.assets
                and string.count("/") >= python_infer_subsystem.assets_min_slashes
            ):
                assets.add(string)

    return ParsedPythonDependencies(
        ParsedPythonImports(
            (key, ParsedPythonImportInfo(*value)) for key, value in imports.items()
        ),
        ParsedPythonAssetPaths(sorted(assets)),
    )


def rules():
    return [
        *collect_rules(),
    ]

1	# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	from __future__ import annotations	12✔
4
5	import logging	12✔
6	import os	12✔
7	from collections.abc import Iterable	12✔
8	from dataclasses import dataclass	12✔
9
10	from pants.backend.python.dependency_inference.subsystem import PythonInferSubsystem	12✔
11	from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints	12✔
12	from pants.core.util_rules.source_files import SourceFiles	12✔
13	from pants.core.util_rules.stripped_source_files import strip_source_roots	12✔
14	from pants.engine.collection import DeduplicatedCollection	12✔
15	from pants.engine.fs import CreateDigest, Digest, FileContent	12✔
16	from pants.engine.internals.native_engine import NativeDependenciesRequest	12✔
17	from pants.engine.intrinsics import create_digest, parse_python_deps	12✔
18	from pants.engine.rules import collect_rules, rule	12✔
19	from pants.util.frozendict import FrozenDict	12✔
20	from pants.util.logging import LogLevel	12✔
21	from pants.util.resources import read_resource	12✔
22
23	logger = logging.getLogger(__name__)	12✔
24
25
26	@dataclass(frozen=True, order=True)	12✔
27	class ParsedPythonImportInfo:	12✔
28	lineno: int	12✔
29	# An import is considered "weak" if we're unsure if a dependency will exist between the parsed
30	# file and the parsed import.
31	# Examples of "weak" imports include string imports (if enabled) or those inside a try block
32	# which has a handler catching ImportError.
33	weak: bool	12✔
34
35
36	class ParsedPythonImports(FrozenDict[str, ParsedPythonImportInfo]):	12✔
37	"""All the discovered imports from a Python source file mapped to the relevant info."""
38
39
40	class ParsedPythonAssetPaths(DeduplicatedCollection[str]):	12✔
41	"""All the discovered possible assets from a Python source file."""
42
43	# N.B. Don't set `sort_input`, as the input is already sorted
44
45
46	@dataclass(frozen=True)	12✔
47	class ParsedPythonDependencies:	12✔
48	imports: ParsedPythonImports	12✔
49	assets: ParsedPythonAssetPaths	12✔
50
51
52	@dataclass(frozen=True)	12✔
53	class ParsePythonDependenciesRequest:	12✔
54	source: SourceFiles	12✔
55	interpreter_constraints: InterpreterConstraints	12✔
56
57
58	@dataclass(frozen=True)	12✔
59	class PythonDependencyVisitor:	12✔
60	"""Wraps a subclass of DependencyVisitorBase."""
61
62	digest: Digest # The file contents for the visitor	12✔
63	classname: str # The full classname, e.g., _my_custom_dep_parser.MyCustomVisitor	12✔
64	env: FrozenDict[str, str] # Set these env vars when invoking the visitor	12✔
65
66
67	@dataclass(frozen=True)	12✔
68	class ParserScript:	12✔
69	digest: Digest	12✔
70	env: FrozenDict[str, str]	12✔
71
72
73	_scripts_package = "pants.backend.python.dependency_inference.scripts"	12✔
74
75
76	async def get_scripts_digest(scripts_package: str, filenames: Iterable[str]) -> Digest:	12✔
77	scripts = [read_resource(scripts_package, filename) for filename in filenames]	×
78	assert all(script is not None for script in scripts)	×
79	path_prefix = scripts_package.replace(".", os.path.sep)	×
80	contents = [	×
81	FileContent(os.path.join(path_prefix, relpath), script)
82	for relpath, script in zip(filenames, scripts)
83	]
84
85	# Python 2 requires all the intermediate __init__.py to exist in the sandbox.
86	package = scripts_package	×
87	while package:	×
88	contents.append(	×
89	FileContent(
90	os.path.join(package.replace(".", os.path.sep), "__init__.py"),
91	read_resource(package, "__init__.py"),
92	)
93	)
94	package = package.rpartition(".")[0]	×
95
96	digest = await create_digest(CreateDigest(contents))	×
97	return digest	×
98
99
100	@rule(level=LogLevel.DEBUG)	12✔
101	async def parse_python_dependencies(	12✔
102	request: ParsePythonDependenciesRequest,
103	python_infer_subsystem: PythonInferSubsystem,
104	) -> ParsedPythonDependencies:
105	stripped_sources = await strip_source_roots(request.source)	×
106	# We operate on PythonSourceField, which should be one file.
107	assert len(stripped_sources.snapshot.files) == 1	×
108
NEW 109	native_results = await parse_python_deps(	×
110	NativeDependenciesRequest(stripped_sources.snapshot.digest)
111	)
NEW 112	assert len(native_results.path_to_deps) == 1	×
NEW 113	native_result = next(iter(native_results.path_to_deps.values()))	×
114	imports = dict(native_result.imports)	×
115	assets = set()	×
116
117	if python_infer_subsystem.string_imports or python_infer_subsystem.assets:	×
118	for string, line in native_result.string_candidates.items():	×
119	if (	×
120	python_infer_subsystem.string_imports
121	and string.count(".") >= python_infer_subsystem.string_imports_min_dots
122	and all(part.isidentifier() for part in string.split("."))
123	):
124	imports.setdefault(string, (line, True))	×
125	if (	×
126	python_infer_subsystem.assets
127	and string.count("/") >= python_infer_subsystem.assets_min_slashes
128	):
129	assets.add(string)	×
130
131	return ParsedPythonDependencies(	×
132	ParsedPythonImports(
133	(key, ParsedPythonImportInfo(*value)) for key, value in imports.items()
134	),
135	ParsedPythonAssetPaths(sorted(assets)),
136	)
137
138
139	def rules():	12✔
140	return [	12✔
141	*collect_rules(),
142	]

pantsbuild / pants / 21726110101

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous