24609520460

Committed 18 Apr 2026 05:01PM UTC coverage: 52.372% (-40.6%) from 92.924%

Build # 24609520460

Build Type

Pull #23268

github

Committed by

web-flow

Commit Message

Merge bd986838e into 0283af69e

Pull Request Pull Request #23268: perf: Remove python coroutine/trampoline overhead in awaits for ~22% faster `dependencies` goal

Coverage Stats

30 of 40 new or added lines in 4 files covered. (75.0%)

23044 existing lines in 605 files now uncovered.

31642 of 60418 relevant lines covered (52.37%)

0.52 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.49

/src/python/pants/backend/python/dependency_inference/parse_python_dependencies.py

# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
from __future__ import annotations

import logging
import os
from collections.abc import Iterable
from dataclasses import dataclass
from fnmatch import fnmatchcase

from pants.backend.python.dependency_inference.subsystem import PythonInferSubsystem
from pants.core.util_rules.source_files import SourceFiles
from pants.core.util_rules.stripped_source_files import strip_source_roots
from pants.engine.collection import DeduplicatedCollection
from pants.engine.fs import CreateDigest, Digest, FileContent
from pants.engine.internals.native_engine import NativeDependenciesRequest
from pants.engine.intrinsics import create_digest, parse_python_deps
from pants.engine.rules import collect_rules, rule
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel
from pants.util.resources import read_resource

logger = logging.getLogger(__name__)


@dataclass(frozen=True, order=True)
class ParsedPythonImportInfo:
    lineno: int
    # An import is considered "weak" if we're unsure if a dependency will exist between the parsed
    # file and the parsed import.
    # Examples of "weak" imports include string imports (if enabled) or those inside a try block
    # which has a handler catching ImportError.
    weak: bool


class ParsedPythonImports(FrozenDict[str, ParsedPythonImportInfo]):
    """All the discovered imports from a Python source file mapped to the relevant info."""


class ParsedPythonAssetPaths(DeduplicatedCollection[str]):
    """All the discovered possible assets from a Python source file."""

    # N.B. Don't set `sort_input`, as the input is already sorted


# Map from argument of `pants: infer-dep(...)` to line number at which it appears.
class ExplicitPythonDependencies(FrozenDict[str, int]):
    """Dependencies provided via the # pants: infer-dep() pragma."""


# TODO: Use the Native* eqivalents of these classes directly? Would require
#  conversion to the component classes in Rust code. Might require passing
#  the PythonInferSubsystem settings through to Rust and acting on them there.


@dataclass(frozen=True)
class PythonFileDependencies:
    imports: ParsedPythonImports
    assets: ParsedPythonAssetPaths
    explicit_dependencies: ExplicitPythonDependencies


@dataclass(frozen=True)
class PythonFilesDependencies:
    path_to_deps: FrozenDict[str, PythonFileDependencies]


@dataclass(frozen=True)
class ParsePythonDependenciesRequest:
    source: SourceFiles


@dataclass(frozen=True)
class PythonDependencyVisitor:
    """Wraps a subclass of DependencyVisitorBase."""

    digest: Digest  # The file contents for the visitor
    classname: str  # The full classname, e.g., _my_custom_dep_parser.MyCustomVisitor
    env: FrozenDict[str, str]  # Set these env vars when invoking the visitor


@dataclass(frozen=True)
class ParserScript:
    digest: Digest
    env: FrozenDict[str, str]


_scripts_package = "pants.backend.python.dependency_inference.scripts"


def _is_ignored_string_import(module_path: str, ignored_patterns: tuple[str, ...]) -> bool:
    return any(fnmatchcase(module_path, ignored_pattern) for ignored_pattern in ignored_patterns)


async def get_scripts_digest(scripts_package: str, filenames: Iterable[str]) -> Digest:
    scripts = [read_resource(scripts_package, filename) for filename in filenames]
    assert all(script is not None for script in scripts)
    path_prefix = scripts_package.replace(".", os.path.sep)
    contents = [
        FileContent(os.path.join(path_prefix, relpath), script)
        for relpath, script in zip(filenames, scripts)
    ]

    # Python 2 requires all the intermediate __init__.py to exist in the sandbox.
    package = scripts_package
    while package:
        contents.append(
            FileContent(
                os.path.join(package.replace(".", os.path.sep), "__init__.py"),
                read_resource(package, "__init__.py"),
            )
        )
        package = package.rpartition(".")[0]

    digest = await create_digest(CreateDigest(contents))
    return digest


@rule(level=LogLevel.DEBUG)
async def parse_python_dependencies(
    request: ParsePythonDependenciesRequest,
    python_infer_subsystem: PythonInferSubsystem,
) -> PythonFilesDependencies:
    stripped_sources = await strip_source_roots(request.source)
    native_results = await parse_python_deps(
        NativeDependenciesRequest(stripped_sources.snapshot.digest)
    )

    path_to_deps = {}
    for path, native_result in native_results.path_to_deps.items():
        imports = dict(native_result.imports)
        assets = set()

        if python_infer_subsystem.string_imports or python_infer_subsystem.assets:
            for string, line in native_result.string_candidates.items():
                if (
                    python_infer_subsystem.string_imports
                    and not _is_ignored_string_import(
                        string, python_infer_subsystem.string_import_ignore
                    )
                    and string.count(".") >= python_infer_subsystem.string_imports_min_dots
                    and all(part.isidentifier() for part in string.split("."))
                ):
                    imports.setdefault(string, (line, True))
                if (
                    python_infer_subsystem.assets
                    and string.count("/") >= python_infer_subsystem.assets_min_slashes
                ):
                    assets.add(string)

        explicit_deps = dict(native_result.explicit_dependencies)

        path_to_deps[path] = PythonFileDependencies(
            ParsedPythonImports(
                (key, ParsedPythonImportInfo(*value)) for key, value in imports.items()
            ),
            ParsedPythonAssetPaths(sorted(assets)),
            ExplicitPythonDependencies(FrozenDict(explicit_deps)),
        )
    return PythonFilesDependencies(FrozenDict(path_to_deps))


def rules():
    return [
        *collect_rules(),
    ]

1	# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	from __future__ import annotations	1✔
4
5	import logging	1✔
6	import os	1✔
7	from collections.abc import Iterable	1✔
8	from dataclasses import dataclass	1✔
9	from fnmatch import fnmatchcase	1✔
10
11	from pants.backend.python.dependency_inference.subsystem import PythonInferSubsystem	1✔
12	from pants.core.util_rules.source_files import SourceFiles	1✔
13	from pants.core.util_rules.stripped_source_files import strip_source_roots	1✔
14	from pants.engine.collection import DeduplicatedCollection	1✔
15	from pants.engine.fs import CreateDigest, Digest, FileContent	1✔
16	from pants.engine.internals.native_engine import NativeDependenciesRequest	1✔
17	from pants.engine.intrinsics import create_digest, parse_python_deps	1✔
18	from pants.engine.rules import collect_rules, rule	1✔
19	from pants.util.frozendict import FrozenDict	1✔
20	from pants.util.logging import LogLevel	1✔
21	from pants.util.resources import read_resource	1✔
22
23	logger = logging.getLogger(__name__)	1✔
24
25
26	@dataclass(frozen=True, order=True)	1✔
27	class ParsedPythonImportInfo:	1✔
28	lineno: int	1✔
29	# An import is considered "weak" if we're unsure if a dependency will exist between the parsed
30	# file and the parsed import.
31	# Examples of "weak" imports include string imports (if enabled) or those inside a try block
32	# which has a handler catching ImportError.
33	weak: bool	1✔
34
35
36	class ParsedPythonImports(FrozenDict[str, ParsedPythonImportInfo]):	1✔
37	"""All the discovered imports from a Python source file mapped to the relevant info."""
38
39
40	class ParsedPythonAssetPaths(DeduplicatedCollection[str]):	1✔
41	"""All the discovered possible assets from a Python source file."""
42
43	# N.B. Don't set `sort_input`, as the input is already sorted
44
45
46	# Map from argument of `pants: infer-dep(...)` to line number at which it appears.
47	class ExplicitPythonDependencies(FrozenDict[str, int]):	1✔
48	"""Dependencies provided via the # pants: infer-dep() pragma."""
49
50
51	# TODO: Use the Native* eqivalents of these classes directly? Would require
52	# conversion to the component classes in Rust code. Might require passing
53	# the PythonInferSubsystem settings through to Rust and acting on them there.
54
55
56	@dataclass(frozen=True)	1✔
57	class PythonFileDependencies:	1✔
58	imports: ParsedPythonImports	1✔
59	assets: ParsedPythonAssetPaths	1✔
60	explicit_dependencies: ExplicitPythonDependencies	1✔
61
62
63	@dataclass(frozen=True)	1✔
64	class PythonFilesDependencies:	1✔
65	path_to_deps: FrozenDict[str, PythonFileDependencies]	1✔
66
67
68	@dataclass(frozen=True)	1✔
69	class ParsePythonDependenciesRequest:	1✔
70	source: SourceFiles	1✔
71
72
73	@dataclass(frozen=True)	1✔
74	class PythonDependencyVisitor:	1✔
75	"""Wraps a subclass of DependencyVisitorBase."""
76
77	digest: Digest # The file contents for the visitor	1✔
78	classname: str # The full classname, e.g., _my_custom_dep_parser.MyCustomVisitor	1✔
79	env: FrozenDict[str, str] # Set these env vars when invoking the visitor	1✔
80
81
82	@dataclass(frozen=True)	1✔
83	class ParserScript:	1✔
84	digest: Digest	1✔
85	env: FrozenDict[str, str]	1✔
86
87
88	_scripts_package = "pants.backend.python.dependency_inference.scripts"	1✔
89
90
91	def _is_ignored_string_import(module_path: str, ignored_patterns: tuple[str, ...]) -> bool:	1✔
UNCOV 92	return any(fnmatchcase(module_path, ignored_pattern) for ignored_pattern in ignored_patterns)	×
93
94
95	async def get_scripts_digest(scripts_package: str, filenames: Iterable[str]) -> Digest:	1✔
96	scripts = [read_resource(scripts_package, filename) for filename in filenames]	×
97	assert all(script is not None for script in scripts)	×
98	path_prefix = scripts_package.replace(".", os.path.sep)	×
99	contents = [	×
100	FileContent(os.path.join(path_prefix, relpath), script)
101	for relpath, script in zip(filenames, scripts)
102	]
103
104	# Python 2 requires all the intermediate __init__.py to exist in the sandbox.
105	package = scripts_package	×
106	while package:	×
107	contents.append(	×
108	FileContent(
109	os.path.join(package.replace(".", os.path.sep), "__init__.py"),
110	read_resource(package, "__init__.py"),
111	)
112	)
113	package = package.rpartition(".")[0]	×
114
115	digest = await create_digest(CreateDigest(contents))	×
116	return digest	×
117
118
119	@rule(level=LogLevel.DEBUG)	1✔
120	async def parse_python_dependencies(	1✔
121	request: ParsePythonDependenciesRequest,
122	python_infer_subsystem: PythonInferSubsystem,
123	) -> PythonFilesDependencies:
124	stripped_sources = await strip_source_roots(request.source)	1✔
125	native_results = await parse_python_deps(	1✔
126	NativeDependenciesRequest(stripped_sources.snapshot.digest)
127	)
128
129	path_to_deps = {}	1✔
130	for path, native_result in native_results.path_to_deps.items():	1✔
131	imports = dict(native_result.imports)	1✔
132	assets = set()	1✔
133
134	if python_infer_subsystem.string_imports or python_infer_subsystem.assets:	1✔
UNCOV 135	for string, line in native_result.string_candidates.items():	×
UNCOV 136	if (	×
137	python_infer_subsystem.string_imports
138	and not _is_ignored_string_import(
139	string, python_infer_subsystem.string_import_ignore
140	)
141	and string.count(".") >= python_infer_subsystem.string_imports_min_dots
142	and all(part.isidentifier() for part in string.split("."))
143	):
UNCOV 144	imports.setdefault(string, (line, True))	×
UNCOV 145	if (	×
146	python_infer_subsystem.assets
147	and string.count("/") >= python_infer_subsystem.assets_min_slashes
148	):
UNCOV 149	assets.add(string)	×
150
151	explicit_deps = dict(native_result.explicit_dependencies)	1✔
152
153	path_to_deps[path] = PythonFileDependencies(	1✔
154	ParsedPythonImports(
155	(key, ParsedPythonImportInfo(*value)) for key, value in imports.items()
156	),
157	ParsedPythonAssetPaths(sorted(assets)),
158	ExplicitPythonDependencies(FrozenDict(explicit_deps)),
159	)
160	return PythonFilesDependencies(FrozenDict(path_to_deps))	1✔
161
162
163	def rules():	1✔
164	return [	1✔
165	*collect_rules(),
166	]

pantsbuild / pants / 24609520460

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous