3671256457

Build Type

push

github

Committed by GitHub

Commit Message

Prepare release 22.12.0 (#3413)

Run Details

6086 of 6351 relevant lines covered (95.83%)

5.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.96

/src/black/parsing.py

"""
Parse Python code and perform AST validation.
"""
import ast
import platform
import sys
from typing import Any, Iterable, Iterator, List, Set, Tuple, Type, Union

if sys.version_info < (3, 8):
    from typing_extensions import Final
else:
    from typing import Final

from black.mode import Feature, TargetVersion, supports_feature
from black.nodes import syms
from blib2to3 import pygram
from blib2to3.pgen2 import driver
from blib2to3.pgen2.grammar import Grammar
from blib2to3.pgen2.parse import ParseError
from blib2to3.pgen2.tokenize import TokenError
from blib2to3.pytree import Leaf, Node

ast3: Any

_IS_PYPY = platform.python_implementation() == "PyPy"

try:
    from typed_ast import ast3
except ImportError:
    if sys.version_info < (3, 8) and not _IS_PYPY:
        print(
            (
                "The typed_ast package is required but not installed.\n"
                "You can upgrade to Python 3.8+ or install typed_ast with\n"
                "`python3 -m pip install typed-ast`."
            ),
            file=sys.stderr,
        )
        sys.exit(1)
    else:
        ast3 = ast


PY2_HINT: Final = "Python 2 support was removed in version 22.0."


class InvalidInput(ValueError):
    """Raised when input source code fails all parse attempts."""


def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:
    if not target_versions:
        # No target_version specified, so try all grammars.
        return [
            # Python 3.7+
            pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords,
            # Python 3.0-3.6
            pygram.python_grammar_no_print_statement_no_exec_statement,
            # Python 3.10+
            pygram.python_grammar_soft_keywords,
        ]

    grammars = []
    # If we have to parse both, try to parse async as a keyword first
    if not supports_feature(
        target_versions, Feature.ASYNC_IDENTIFIERS
    ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):
        # Python 3.7-3.9
        grammars.append(
            pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords
        )
    if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):
        # Python 3.0-3.6
        grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)
    if supports_feature(target_versions, Feature.PATTERN_MATCHING):
        # Python 3.10+
        grammars.append(pygram.python_grammar_soft_keywords)

    # At least one of the above branches must have been taken, because every Python
    # version has exactly one of the two 'ASYNC_*' flags
    return grammars


def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:
    """Given a string with source, return the lib2to3 Node."""
    if not src_txt.endswith("\n"):
        src_txt += "\n"

    grammars = get_grammars(set(target_versions))
    errors = {}
    for grammar in grammars:
        drv = driver.Driver(grammar)
        try:
            result = drv.parse_string(src_txt, True)
            break

        except ParseError as pe:
            lineno, column = pe.context[1]
            lines = src_txt.splitlines()
            try:
                faulty_line = lines[lineno - 1]
            except IndexError:
                faulty_line = "<line number missing in source>"
            errors[grammar.version] = InvalidInput(
                f"Cannot parse: {lineno}:{column}: {faulty_line}"
            )

        except TokenError as te:
            # In edge cases these are raised; and typically don't have a "faulty_line".
            lineno, column = te.args[1]
            errors[grammar.version] = InvalidInput(
                f"Cannot parse: {lineno}:{column}: {te.args[0]}"
            )

    else:
        # Choose the latest version when raising the actual parsing error.
        assert len(errors) >= 1
        exc = errors[max(errors)]

        if matches_grammar(src_txt, pygram.python_grammar) or matches_grammar(
            src_txt, pygram.python_grammar_no_print_statement
        ):
            original_msg = exc.args[0]
            msg = f"{original_msg}\n{PY2_HINT}"
            raise InvalidInput(msg) from None

        raise exc from None

    if isinstance(result, Leaf):
        result = Node(syms.file_input, [result])
    return result


def matches_grammar(src_txt: str, grammar: Grammar) -> bool:
    drv = driver.Driver(grammar)
    try:
        drv.parse_string(src_txt, True)
    except (ParseError, TokenError, IndentationError):
        return False
    else:
        return True


def lib2to3_unparse(node: Node) -> str:
    """Given a lib2to3 node, return its string representation."""
    code = str(node)
    return code


def parse_single_version(
    src: str, version: Tuple[int, int]
) -> Union[ast.AST, ast3.AST]:
    filename = "<unknown>"
    # typed-ast is needed because of feature version limitations in the builtin ast 3.8>
    if sys.version_info >= (3, 8) and version >= (3,):
        return ast.parse(src, filename, feature_version=version, type_comments=True)

    if _IS_PYPY:
        # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's
        # not much we can do as typed-ast won't work either.
        if sys.version_info >= (3, 8):
            return ast3.parse(src, filename, type_comments=True)
        else:
            return ast3.parse(src, filename)
    else:
        # Typed-ast is guaranteed to be used here and automatically tracks type
        # comments separately.
        return ast3.parse(src, filename, feature_version=version[1])


def parse_ast(src: str) -> Union[ast.AST, ast3.AST]:
    # TODO: support Python 4+ ;)
    versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)]

    first_error = ""
    for version in sorted(versions, reverse=True):
        try:
            return parse_single_version(src, version)
        except SyntaxError as e:
            if not first_error:
                first_error = str(e)

    raise SyntaxError(first_error)


ast3_AST: Final[Type[ast3.AST]] = ast3.AST


def _normalize(lineend: str, value: str) -> str:
    # To normalize, we strip any leading and trailing space from
    # each line...
    stripped: List[str] = [i.strip() for i in value.splitlines()]
    normalized = lineend.join(stripped)
    # ...and remove any blank lines at the beginning and end of
    # the whole string
    return normalized.strip()


def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[str]:
    """Simple visitor generating strings to compare ASTs by content."""

    node = fixup_ast_constants(node)

    yield f"{'  ' * depth}{node.__class__.__name__}("

    type_ignore_classes: Tuple[Type[Any], ...]
    for field in sorted(node._fields):  # noqa: F402
        # TypeIgnore will not be present using pypy < 3.8, so need for this
        if not (_IS_PYPY and sys.version_info < (3, 8)):
            # TypeIgnore has only one field 'lineno' which breaks this comparison
            type_ignore_classes = (ast3.TypeIgnore,)
            if sys.version_info >= (3, 8):
                type_ignore_classes += (ast.TypeIgnore,)
            if isinstance(node, type_ignore_classes):
                break

        try:
            value: object = getattr(node, field)
        except AttributeError:
            continue

        yield f"{'  ' * (depth+1)}{field}="

        if isinstance(value, list):
            for item in value:
                # Ignore nested tuples within del statements, because we may insert
                # parentheses and they change the AST.
                if (
                    field == "targets"
                    and isinstance(node, (ast.Delete, ast3.Delete))
                    and isinstance(item, (ast.Tuple, ast3.Tuple))
                ):
                    for elt in item.elts:
                        yield from stringify_ast(elt, depth + 2)

                elif isinstance(item, (ast.AST, ast3.AST)):
                    yield from stringify_ast(item, depth + 2)

        # Note that we are referencing the typed-ast ASTs via global variables and not
        # direct module attribute accesses because that breaks mypyc. It's probably
        # something to do with the ast3 variables being marked as Any leading
        # mypy to think this branch is always taken, leaving the rest of the code
        # unanalyzed. Tighting up the types for the typed-ast AST types avoids the
        # mypyc crash.
        elif isinstance(value, (ast.AST, ast3_AST)):
            yield from stringify_ast(value, depth + 2)

        else:
            normalized: object
            # Constant strings may be indented across newlines, if they are
            # docstrings; fold spaces after newlines when comparing. Similarly,
            # trailing and leading space may be removed.
            if (
                isinstance(node, ast.Constant)
                and field == "value"
                and isinstance(value, str)
            ):
                normalized = _normalize("\n", value)
            else:
                normalized = value
            yield f"{'  ' * (depth+2)}{normalized!r},  # {value.__class__.__name__}"

    yield f"{'  ' * depth})  # /{node.__class__.__name__}"


def fixup_ast_constants(node: Union[ast.AST, ast3.AST]) -> Union[ast.AST, ast3.AST]:
    """Map ast nodes deprecated in 3.8 to Constant."""
    if isinstance(node, (ast.Str, ast3.Str, ast.Bytes, ast3.Bytes)):
        return ast.Constant(value=node.s)

    if isinstance(node, (ast.Num, ast3.Num)):
        return ast.Constant(value=node.n)

    if isinstance(node, (ast.NameConstant, ast3.NameConstant)):
        return ast.Constant(value=node.value)

    return node

1	"""
2	Parse Python code and perform AST validation.
3	"""
4	import ast	6✔
5	import platform	6✔
6	import sys	6✔
7	from typing import Any, Iterable, Iterator, List, Set, Tuple, Type, Union	6✔
8
9	if sys.version_info < (3, 8):	6✔
10	from typing_extensions import Final	2✔
11	else:
12	from typing import Final	4✔
13
14	from black.mode import Feature, TargetVersion, supports_feature	6✔
15	from black.nodes import syms	6✔
16	from blib2to3 import pygram	6✔
17	from blib2to3.pgen2 import driver	6✔
18	from blib2to3.pgen2.grammar import Grammar	6✔
19	from blib2to3.pgen2.parse import ParseError	6✔
20	from blib2to3.pgen2.tokenize import TokenError	6✔
21	from blib2to3.pytree import Leaf, Node	6✔
22
23	ast3: Any	6✔
24
25	_IS_PYPY = platform.python_implementation() == "PyPy"	6✔
26
27	try:	6✔
28	from typed_ast import ast3	6✔
29	except ImportError:	5✔
30	if sys.version_info < (3, 8) and not _IS_PYPY:	5✔
31	print(	×
32	(
33	"The typed_ast package is required but not installed.\n"
34	"You can upgrade to Python 3.8+ or install typed_ast with\n"
35	"`python3 -m pip install typed-ast`."
36	),
37	file=sys.stderr,
38	)
39	sys.exit(1)	×
40	else:
41	ast3 = ast	5✔
42
43
44	PY2_HINT: Final = "Python 2 support was removed in version 22.0."	6✔
45
46
47	class InvalidInput(ValueError):	6✔
48	"""Raised when input source code fails all parse attempts."""
49
50
51	def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]:	6✔
52	if not target_versions:	6✔
53	# No target_version specified, so try all grammars.
54	return [	6✔
55	# Python 3.7+
56	pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords,
57	# Python 3.0-3.6
58	pygram.python_grammar_no_print_statement_no_exec_statement,
59	# Python 3.10+
60	pygram.python_grammar_soft_keywords,
61	]
62
63	grammars = []	6✔
64	# If we have to parse both, try to parse async as a keyword first
65	if not supports_feature(	6✔
66	target_versions, Feature.ASYNC_IDENTIFIERS
67	) and not supports_feature(target_versions, Feature.PATTERN_MATCHING):
68	# Python 3.7-3.9
69	grammars.append(	6✔
70	pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords
71	)
72	if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS):	6✔
73	# Python 3.0-3.6
74	grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement)	6✔
75	if supports_feature(target_versions, Feature.PATTERN_MATCHING):	6✔
76	# Python 3.10+
77	grammars.append(pygram.python_grammar_soft_keywords)	6✔
78
79	# At least one of the above branches must have been taken, because every Python
80	# version has exactly one of the two 'ASYNC_*' flags
81	return grammars	6✔
82
83
84	def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) -> Node:	6✔
85	"""Given a string with source, return the lib2to3 Node."""
86	if not src_txt.endswith("\n"):	6✔
87	src_txt += "\n"	6✔
88
89	grammars = get_grammars(set(target_versions))	6✔
90	errors = {}	6✔
91	for grammar in grammars:	6✔
92	drv = driver.Driver(grammar)	6✔
93	try:	6✔
94	result = drv.parse_string(src_txt, True)	6✔
95	break	6✔
96
97	except ParseError as pe:	6✔
98	lineno, column = pe.context[1]	6✔
99	lines = src_txt.splitlines()	6✔
100	try:	6✔
101	faulty_line = lines[lineno - 1]	6✔
102	except IndexError:	×
103	faulty_line = "<line number missing in source>"	×
104	errors[grammar.version] = InvalidInput(	6✔
105	f"Cannot parse: {lineno}:{column}: {faulty_line}"
106	)
107
108	except TokenError as te:	6✔
109	# In edge cases these are raised; and typically don't have a "faulty_line".
110	lineno, column = te.args[1]	6✔
111	errors[grammar.version] = InvalidInput(	6✔
112	f"Cannot parse: {lineno}:{column}: {te.args[0]}"
113	)
114
115	else:
116	# Choose the latest version when raising the actual parsing error.
117	assert len(errors) >= 1	6✔
118	exc = errors[max(errors)]	6✔
119
120	if matches_grammar(src_txt, pygram.python_grammar) or matches_grammar(	6✔
121	src_txt, pygram.python_grammar_no_print_statement
122	):
123	original_msg = exc.args[0]	6✔
124	msg = f"{original_msg}\n{PY2_HINT}"	6✔
125	raise InvalidInput(msg) from None	6✔
126
127	raise exc from None	6✔
128
129	if isinstance(result, Leaf):	6✔
130	result = Node(syms.file_input, [result])	6✔
131	return result	6✔
132
133
134	def matches_grammar(src_txt: str, grammar: Grammar) -> bool:	6✔
135	drv = driver.Driver(grammar)	6✔
136	try:	6✔
137	drv.parse_string(src_txt, True)	6✔
138	except (ParseError, TokenError, IndentationError):	6✔
139	return False	6✔
140	else:
141	return True	6✔
142
143
144	def lib2to3_unparse(node: Node) -> str:	6✔
145	"""Given a lib2to3 node, return its string representation."""
146	code = str(node)	×
147	return code	×
148
149
150	def parse_single_version(	6✔
151	src: str, version: Tuple[int, int]
152	) -> Union[ast.AST, ast3.AST]:
153	filename = "<unknown>"	6✔
154	# typed-ast is needed because of feature version limitations in the builtin ast 3.8>
155	if sys.version_info >= (3, 8) and version >= (3,):	6✔
156	return ast.parse(src, filename, feature_version=version, type_comments=True)	4✔
157
158	if _IS_PYPY:	2✔
159	# PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's
160	# not much we can do as typed-ast won't work either.
161	if sys.version_info >= (3, 8):	1✔
162	return ast3.parse(src, filename, type_comments=True)	×
163	else:
164	return ast3.parse(src, filename)	1✔
165	else:
166	# Typed-ast is guaranteed to be used here and automatically tracks type
167	# comments separately.
168	return ast3.parse(src, filename, feature_version=version[1])	1✔
169
170
171	def parse_ast(src: str) -> Union[ast.AST, ast3.AST]:	6✔
172	# TODO: support Python 4+ ;)
173	versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)]	6✔
174
175	first_error = ""	6✔
176	for version in sorted(versions, reverse=True):	6✔
177	try:	6✔
178	return parse_single_version(src, version)	6✔
179	except SyntaxError as e:	6✔
180	if not first_error:	6✔
181	first_error = str(e)	6✔
182
183	raise SyntaxError(first_error)	6✔
184
185
186	ast3_AST: Final[Type[ast3.AST]] = ast3.AST	6✔
187
188
189	def _normalize(lineend: str, value: str) -> str:	6✔
190	# To normalize, we strip any leading and trailing space from
191	# each line...
192	stripped: List[str] = [i.strip() for i in value.splitlines()]	6✔
193	normalized = lineend.join(stripped)	6✔
194	# ...and remove any blank lines at the beginning and end of
195	# the whole string
196	return normalized.strip()	6✔
197
198
199	def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[str]:	6✔
200	"""Simple visitor generating strings to compare ASTs by content."""
201
202	node = fixup_ast_constants(node)	6✔
203
204	yield f"{' ' * depth}{node.__class__.__name__}("	6✔
205
206	type_ignore_classes: Tuple[Type[Any], ...]
207	for field in sorted(node._fields): # noqa: F402	6✔
208	# TypeIgnore will not be present using pypy < 3.8, so need for this
209	if not (_IS_PYPY and sys.version_info < (3, 8)):	6✔
210	# TypeIgnore has only one field 'lineno' which breaks this comparison
211	type_ignore_classes = (ast3.TypeIgnore,)	5✔
212	if sys.version_info >= (3, 8):	5✔
213	type_ignore_classes += (ast.TypeIgnore,)	4✔
214	if isinstance(node, type_ignore_classes):	5✔
215	break	5✔
216
217	try:	6✔
218	value: object = getattr(node, field)	6✔
219	except AttributeError:	2✔
220	continue	2✔
221
222	yield f"{' ' * (depth+1)}{field}="	6✔
223
224	if isinstance(value, list):	6✔
225	for item in value:	6✔
226	# Ignore nested tuples within del statements, because we may insert
227	# parentheses and they change the AST.
228	if (	6✔
229	field == "targets"
230	and isinstance(node, (ast.Delete, ast3.Delete))
231	and isinstance(item, (ast.Tuple, ast3.Tuple))
232	):
233	for elt in item.elts:	6✔
234	yield from stringify_ast(elt, depth + 2)	6✔
235
236	elif isinstance(item, (ast.AST, ast3.AST)):	6✔
237	yield from stringify_ast(item, depth + 2)	6✔
238
239	# Note that we are referencing the typed-ast ASTs via global variables and not
240	# direct module attribute accesses because that breaks mypyc. It's probably
241	# something to do with the ast3 variables being marked as Any leading
242	# mypy to think this branch is always taken, leaving the rest of the code
243	# unanalyzed. Tighting up the types for the typed-ast AST types avoids the
244	# mypyc crash.
245	elif isinstance(value, (ast.AST, ast3_AST)):	6✔
246	yield from stringify_ast(value, depth + 2)	6✔
247
248	else:
249	normalized: object
250	# Constant strings may be indented across newlines, if they are
251	# docstrings; fold spaces after newlines when comparing. Similarly,
252	# trailing and leading space may be removed.
253	if (	6✔
254	isinstance(node, ast.Constant)
255	and field == "value"
256	and isinstance(value, str)
257	):
258	normalized = _normalize("\n", value)	6✔
259	else:
260	normalized = value	6✔
261	yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}"	6✔
262
263	yield f"{' ' * depth}) # /{node.__class__.__name__}"	6✔
264
265
266	def fixup_ast_constants(node: Union[ast.AST, ast3.AST]) -> Union[ast.AST, ast3.AST]:	6✔
267	"""Map ast nodes deprecated in 3.8 to Constant."""
268	if isinstance(node, (ast.Str, ast3.Str, ast.Bytes, ast3.Bytes)):	6✔
269	return ast.Constant(value=node.s)	6✔
270
271	if isinstance(node, (ast.Num, ast3.Num)):	6✔
272	return ast.Constant(value=node.n)	6✔
273
274	if isinstance(node, (ast.NameConstant, ast3.NameConstant)):	6✔
275	return ast.Constant(value=node.value)	6✔
276
277	return node	6✔

psf / black / 3671256457

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous