36073

pending completion

Build # 36073

Build Type

push

travis-ci-com

Committed by

web-flow

Commit Message

Revert using libCST for dep inference due to performance (#10907) (#11001)

Unfortunately, libCST resulted in a major slowdown.

Before, ff02ef0dd:

```
▶ /usr/bin/time ./pants --no-pantsd typecheck src/python/pants::
...
       10.63 real        11.47 user         6.99 sys
```

After, ea542ac16:

```
▶ /usr/bin/time ./pants --no-pantsd typecheck src/python/pants::
...
       35.40 real        36.13 user         7.08 sys
```

This makes sense: libCST stores a Concrete Syntax Tree, rather than an AST. It's doing more work, like preserving whitespace.

Unfortunately, this means that you must again run Pants with Python 3.8 in order to parse Python 3.8-only syntax. Because the original fix results in a 240% slowdown, this is less offensive than the slowdown, even though it's not ideal. 

[ci skip-rust]
[ci skip-build-wheels]

Run Details

42 of 42 new or added lines in 2 files covered. (100.0%)

0 of 29394 relevant lines covered (0.0%)

0.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/python/pants/backend/python/dependency_inference/import_parser.py

# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import ast as ast3
import re
import sys
from dataclasses import dataclass
from typing import Optional, Set, Tuple

from typed_ast import ast27

from pants.util.memo import memoized_property
from pants.util.ordered_set import FrozenOrderedSet
from pants.util.strutil import ensure_text


class ImportParseError(ValueError):
    pass


@dataclass(frozen=True)
class ParsedPythonImports:
    """All the discovered imports from a Python source file.

    Explicit imports are imports from `import x` and `from module import x` statements. Inferred
    imports come from strings that look like module names, such as
    `importlib.import_module("example.subdir.Foo")`.
    """

    explicit_imports: FrozenOrderedSet[str]
    inferred_imports: FrozenOrderedSet[str]

    @memoized_property
    def all_imports(self) -> FrozenOrderedSet[str]:
        return FrozenOrderedSet(sorted([*self.explicit_imports, *self.inferred_imports]))


def parse_file(*, filename: str, content: str) -> Optional[Tuple]:
    try:
        # NB: The Python 3 ast is generally backwards-compatible with earlier versions. The only
        # breaking change is `async` `await` becoming reserved keywords in Python 3.7 (deprecated
        # in 3.6). If the std-lib fails to parse, we could use typed-ast to try parsing with a
        # target version of Python 3.5, but we don't because Python 3.5 is almost EOL and has very
        # low usage.
        # We will also fail to parse Python 3.8 syntax if Pants is run with Python 3.6 or 3.7.
        # There is no known workaround for this, beyond users changing their `./pants` script to
        # always use >= 3.8.
        tree = ast3.parse(content, filename=filename)
        visitor_cls = _Py3AstVisitor if sys.version_info[:2] < (3, 8) else _Py38AstVisitor
        return tree, visitor_cls
    except SyntaxError:
        try:
            py27_tree = ast27.parse(content, filename=filename)
            return py27_tree, _Py27AstVisitor
        except SyntaxError:
            return None


def find_python_imports(*, filename: str, content: str, module_name: str) -> ParsedPythonImports:
    parse_result = parse_file(filename=filename, content=content)
    # If there were syntax errors, gracefully early return. This is more user friendly than
    # propagating the exception. Dependency inference simply won't be used for that file, and
    # it'll be up to the tool actually being run (e.g. Pytest or Flake8) to error.
    if parse_result is None:
        return ParsedPythonImports(FrozenOrderedSet(), FrozenOrderedSet())
    tree, ast_visitor_cls = parse_result
    ast_visitor = ast_visitor_cls(module_name)
    ast_visitor.visit(tree)
    return ParsedPythonImports(
        explicit_imports=FrozenOrderedSet(sorted(ast_visitor.explicit_imports)),
        inferred_imports=FrozenOrderedSet(sorted(ast_visitor.inferred_imports)),
    )


# This regex is used to infer imports from strings, e.g.
#  `importlib.import_module("example.subdir.Foo")`.
_INFERRED_IMPORT_REGEX = re.compile(r"^([a-z_][a-z_\d]*\.){2,}[a-zA-Z_]\w*$")


class _BaseAstVisitor:
    def __init__(self, module_name: str) -> None:
        self._module_parts = module_name.split(".")
        self.explicit_imports: Set[str] = set()
        self.inferred_imports: Set[str] = set()

    def maybe_add_inferred_import(self, s: str) -> None:
        if _INFERRED_IMPORT_REGEX.match(s):
            self.inferred_imports.add(s)

    def visit_Import(self, node) -> None:
        for alias in node.names:
            self.explicit_imports.add(alias.name)

    def visit_ImportFrom(self, node) -> None:
        rel_module = node.module
        abs_module = ".".join(
            self._module_parts[0 : -node.level] + ([] if rel_module is None else [rel_module])
        )
        for alias in node.names:
            self.explicit_imports.add(f"{abs_module}.{alias.name}")


class _Py27AstVisitor(ast27.NodeVisitor, _BaseAstVisitor):
    def visit_Str(self, node) -> None:
        val = ensure_text(node.s)
        self.maybe_add_inferred_import(val)


class _Py3AstVisitor(ast3.NodeVisitor, _BaseAstVisitor):
    def visit_Str(self, node) -> None:
        self.maybe_add_inferred_import(node.s)


class _Py38AstVisitor(ast3.NodeVisitor, _BaseAstVisitor):
    # Python 3.8 deprecated the Str node in favor of Constant.
    def visit_Constant(self, node) -> None:
        if isinstance(node.value, str):
            self.maybe_add_inferred_import(node.value)

1	# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	import ast as ast3	×
5	import re	×
6	import sys	×
7	from dataclasses import dataclass	×
8	from typing import Optional, Set, Tuple	×
9
10	from typed_ast import ast27	×
11
12	from pants.util.memo import memoized_property	×
13	from pants.util.ordered_set import FrozenOrderedSet	×
14	from pants.util.strutil import ensure_text	×
15
16
17	class ImportParseError(ValueError):	×
18	pass	×
19
20
21	@dataclass(frozen=True)	×
22	class ParsedPythonImports:
23	"""All the discovered imports from a Python source file.
24
25	Explicit imports are imports from `import x` and `from module import x` statements. Inferred
26	imports come from strings that look like module names, such as
27	`importlib.import_module("example.subdir.Foo")`.
28	"""
29
30	explicit_imports: FrozenOrderedSet[str]	×
31	inferred_imports: FrozenOrderedSet[str]	×
32
33	@memoized_property	×
34	def all_imports(self) -> FrozenOrderedSet[str]:	×
35	return FrozenOrderedSet(sorted([self.explicit_imports, self.inferred_imports]))	×
36
37
38	def parse_file(*, filename: str, content: str) -> Optional[Tuple]:	×
39	try:	×
40	# NB: The Python 3 ast is generally backwards-compatible with earlier versions. The only
41	# breaking change is `async` `await` becoming reserved keywords in Python 3.7 (deprecated
42	# in 3.6). If the std-lib fails to parse, we could use typed-ast to try parsing with a
43	# target version of Python 3.5, but we don't because Python 3.5 is almost EOL and has very
44	# low usage.
45	# We will also fail to parse Python 3.8 syntax if Pants is run with Python 3.6 or 3.7.
46	# There is no known workaround for this, beyond users changing their `./pants` script to
47	# always use >= 3.8.
48	tree = ast3.parse(content, filename=filename)	×
49	visitor_cls = _Py3AstVisitor if sys.version_info[:2] < (3, 8) else _Py38AstVisitor	×
50	return tree, visitor_cls	×
51	except SyntaxError:	×
52	try:	×
53	py27_tree = ast27.parse(content, filename=filename)	×
54	return py27_tree, _Py27AstVisitor	×
55	except SyntaxError:	×
56	return None	×
57
58
59	def find_python_imports(*, filename: str, content: str, module_name: str) -> ParsedPythonImports:	×
60	parse_result = parse_file(filename=filename, content=content)	×
61	# If there were syntax errors, gracefully early return. This is more user friendly than
62	# propagating the exception. Dependency inference simply won't be used for that file, and
63	# it'll be up to the tool actually being run (e.g. Pytest or Flake8) to error.
64	if parse_result is None:	×
65	return ParsedPythonImports(FrozenOrderedSet(), FrozenOrderedSet())	×
66	tree, ast_visitor_cls = parse_result	×
67	ast_visitor = ast_visitor_cls(module_name)	×
68	ast_visitor.visit(tree)	×
69	return ParsedPythonImports(	×
70	explicit_imports=FrozenOrderedSet(sorted(ast_visitor.explicit_imports)),
71	inferred_imports=FrozenOrderedSet(sorted(ast_visitor.inferred_imports)),
72	)
73
74
75	# This regex is used to infer imports from strings, e.g.
76	# `importlib.import_module("example.subdir.Foo")`.
77	_INFERRED_IMPORT_REGEX = re.compile(r"^([a-z_][a-z_\d]\.){2,}[a-zA-Z_]\w$")	×
78
79
80	class _BaseAstVisitor:	×
81	def __init__(self, module_name: str) -> None:	×
82	self._module_parts = module_name.split(".")	×
83	self.explicit_imports: Set[str] = set()	×
84	self.inferred_imports: Set[str] = set()	×
85
86	def maybe_add_inferred_import(self, s: str) -> None:	×
87	if _INFERRED_IMPORT_REGEX.match(s):	×
88	self.inferred_imports.add(s)	×
89
90	def visit_Import(self, node) -> None:	×
91	for alias in node.names:	×
92	self.explicit_imports.add(alias.name)	×
93
94	def visit_ImportFrom(self, node) -> None:	×
95	rel_module = node.module	×
96	abs_module = ".".join(	×
97	self._module_parts[0 : -node.level] + ([] if rel_module is None else [rel_module])
98	)
99	for alias in node.names:	×
100	self.explicit_imports.add(f"{abs_module}.{alias.name}")	×
101
102
103	class _Py27AstVisitor(ast27.NodeVisitor, _BaseAstVisitor):	×
104	def visit_Str(self, node) -> None:	×
105	val = ensure_text(node.s)	×
106	self.maybe_add_inferred_import(val)	×
107
108
109	class _Py3AstVisitor(ast3.NodeVisitor, _BaseAstVisitor):	×
110	def visit_Str(self, node) -> None:	×
111	self.maybe_add_inferred_import(node.s)	×
112
113
114	class _Py38AstVisitor(ast3.NodeVisitor, _BaseAstVisitor):	×
115	# Python 3.8 deprecated the Str node in favor of Constant.
116	def visit_Constant(self, node) -> None:	×
117	if isinstance(node.value, str):	×
118	self.maybe_add_inferred_import(node.value)	×

pantsbuild / pants / 36073

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous