| """Tests for the unparse.py script in the Tools/parser directory.""" |
| |
| import unittest |
| import test.support |
| import pathlib |
| import random |
| import tokenize |
| import ast |
| |
| |
| def read_pyfile(filename): |
| """Read and return the contents of a Python source file (as a |
| string), taking into account the file encoding.""" |
| with tokenize.open(filename) as stream: |
| return stream.read() |
| |
| |
| for_else = """\ |
| def f(): |
| for x in range(10): |
| break |
| else: |
| y = 2 |
| z = 3 |
| """ |
| |
| while_else = """\ |
| def g(): |
| while True: |
| break |
| else: |
| y = 2 |
| z = 3 |
| """ |
| |
| relative_import = """\ |
| from . import fred |
| from .. import barney |
| from .australia import shrimp as prawns |
| """ |
| |
| nonlocal_ex = """\ |
| def f(): |
| x = 1 |
| def g(): |
| nonlocal x |
| x = 2 |
| y = 7 |
| def h(): |
| nonlocal x, y |
| """ |
| |
| # also acts as test for 'except ... as ...' |
| raise_from = """\ |
| try: |
| 1 / 0 |
| except ZeroDivisionError as e: |
| raise ArithmeticError from e |
| """ |
| |
| class_decorator = """\ |
| @f1(arg) |
| @f2 |
| class Foo: pass |
| """ |
| |
| elif1 = """\ |
| if cond1: |
| suite1 |
| elif cond2: |
| suite2 |
| else: |
| suite3 |
| """ |
| |
| elif2 = """\ |
| if cond1: |
| suite1 |
| elif cond2: |
| suite2 |
| """ |
| |
| try_except_finally = """\ |
| try: |
| suite1 |
| except ex1: |
| suite2 |
| except ex2: |
| suite3 |
| else: |
| suite4 |
| finally: |
| suite5 |
| """ |
| |
| with_simple = """\ |
| with f(): |
| suite1 |
| """ |
| |
| with_as = """\ |
| with f() as x: |
| suite1 |
| """ |
| |
| with_two_items = """\ |
| with f() as x, g() as y: |
| suite1 |
| """ |
| |
| docstring_prefixes = ( |
| "", |
| "class foo:\n ", |
| "def foo():\n ", |
| "async def foo():\n ", |
| ) |
| |
| class ASTTestCase(unittest.TestCase): |
| def assertASTEqual(self, ast1, ast2): |
| self.assertEqual(ast.dump(ast1), ast.dump(ast2)) |
| |
| def check_ast_roundtrip(self, code1, **kwargs): |
| with self.subTest(code1=code1, ast_parse_kwargs=kwargs): |
| ast1 = ast.parse(code1, **kwargs) |
| code2 = ast.unparse(ast1) |
| ast2 = ast.parse(code2, **kwargs) |
| self.assertASTEqual(ast1, ast2) |
| |
| def check_invalid(self, node, raises=ValueError): |
| with self.subTest(node=node): |
| self.assertRaises(raises, ast.unparse, node) |
| |
| def get_source(self, code1, code2=None): |
| code2 = code2 or code1 |
| code1 = ast.unparse(ast.parse(code1)) |
| return code1, code2 |
| |
| def check_src_roundtrip(self, code1, code2=None): |
| code1, code2 = self.get_source(code1, code2) |
| with self.subTest(code1=code1, code2=code2): |
| self.assertEqual(code2, code1) |
| |
| def check_src_dont_roundtrip(self, code1, code2=None): |
| code1, code2 = self.get_source(code1, code2) |
| with self.subTest(code1=code1, code2=code2): |
| self.assertNotEqual(code2, code1) |
| |
| class UnparseTestCase(ASTTestCase): |
| # Tests for specific bugs found in earlier versions of unparse |
| |
| def test_fstrings(self): |
| # See issue 25180 |
| self.check_ast_roundtrip(r"""f'{f"{0}"*3}'""") |
| self.check_ast_roundtrip(r"""f'{f"{y}"*3}'""") |
| self.check_ast_roundtrip("""f''""") |
| self.check_ast_roundtrip('''f"""'end' "quote\\""""''') |
| |
| def test_fstrings_complicated(self): |
| # See issue 28002 |
| self.check_ast_roundtrip("""f'''{"'"}'''""") |
| self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') |
| self.check_ast_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-'single quote\\'\'\'\'''') |
| self.check_ast_roundtrip('f"""{\'\'\'\n\'\'\'}"""') |
| self.check_ast_roundtrip('f"""{g(\'\'\'\n\'\'\')}"""') |
| self.check_ast_roundtrip('''f"a\\r\\nb"''') |
| self.check_ast_roundtrip('''f"\\u2028{'x'}"''') |
| |
| def test_strings(self): |
| self.check_ast_roundtrip("u'foo'") |
| self.check_ast_roundtrip("r'foo'") |
| self.check_ast_roundtrip("b'foo'") |
| |
| def test_del_statement(self): |
| self.check_ast_roundtrip("del x, y, z") |
| |
| def test_shifts(self): |
| self.check_ast_roundtrip("45 << 2") |
| self.check_ast_roundtrip("13 >> 7") |
| |
| def test_for_else(self): |
| self.check_ast_roundtrip(for_else) |
| |
| def test_while_else(self): |
| self.check_ast_roundtrip(while_else) |
| |
| def test_unary_parens(self): |
| self.check_ast_roundtrip("(-1)**7") |
| self.check_ast_roundtrip("(-1.)**8") |
| self.check_ast_roundtrip("(-1j)**6") |
| self.check_ast_roundtrip("not True or False") |
| self.check_ast_roundtrip("True or not False") |
| |
| def test_integer_parens(self): |
| self.check_ast_roundtrip("3 .__abs__()") |
| |
| def test_huge_float(self): |
| self.check_ast_roundtrip("1e1000") |
| self.check_ast_roundtrip("-1e1000") |
| self.check_ast_roundtrip("1e1000j") |
| self.check_ast_roundtrip("-1e1000j") |
| |
| def test_nan(self): |
| self.assertASTEqual( |
| ast.parse(ast.unparse(ast.Constant(value=float('nan')))), |
| ast.parse('1e1000 - 1e1000') |
| ) |
| |
| def test_min_int(self): |
| self.check_ast_roundtrip(str(-(2 ** 31))) |
| self.check_ast_roundtrip(str(-(2 ** 63))) |
| |
| def test_imaginary_literals(self): |
| self.check_ast_roundtrip("7j") |
| self.check_ast_roundtrip("-7j") |
| self.check_ast_roundtrip("0j") |
| self.check_ast_roundtrip("-0j") |
| |
| def test_lambda_parentheses(self): |
| self.check_ast_roundtrip("(lambda: int)()") |
| |
| def test_chained_comparisons(self): |
| self.check_ast_roundtrip("1 < 4 <= 5") |
| self.check_ast_roundtrip("a is b is c is not d") |
| |
| def test_function_arguments(self): |
| self.check_ast_roundtrip("def f(): pass") |
| self.check_ast_roundtrip("def f(a): pass") |
| self.check_ast_roundtrip("def f(b = 2): pass") |
| self.check_ast_roundtrip("def f(a, b): pass") |
| self.check_ast_roundtrip("def f(a, b = 2): pass") |
| self.check_ast_roundtrip("def f(a = 5, b = 2): pass") |
| self.check_ast_roundtrip("def f(*, a = 1, b = 2): pass") |
| self.check_ast_roundtrip("def f(*, a = 1, b): pass") |
| self.check_ast_roundtrip("def f(*, a, b = 2): pass") |
| self.check_ast_roundtrip("def f(a, b = None, *, c, **kwds): pass") |
| self.check_ast_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass") |
| self.check_ast_roundtrip("def f(*args, **kwargs): pass") |
| |
| def test_relative_import(self): |
| self.check_ast_roundtrip(relative_import) |
| |
| def test_nonlocal(self): |
| self.check_ast_roundtrip(nonlocal_ex) |
| |
| def test_raise_from(self): |
| self.check_ast_roundtrip(raise_from) |
| |
| def test_bytes(self): |
| self.check_ast_roundtrip("b'123'") |
| |
| def test_annotations(self): |
| self.check_ast_roundtrip("def f(a : int): pass") |
| self.check_ast_roundtrip("def f(a: int = 5): pass") |
| self.check_ast_roundtrip("def f(*args: [int]): pass") |
| self.check_ast_roundtrip("def f(**kwargs: dict): pass") |
| self.check_ast_roundtrip("def f() -> None: pass") |
| |
| def test_set_literal(self): |
| self.check_ast_roundtrip("{'a', 'b', 'c'}") |
| |
| def test_empty_set(self): |
| self.assertASTEqual( |
| ast.parse(ast.unparse(ast.Set(elts=[]))), |
| ast.parse('{*()}') |
| ) |
| |
| def test_set_comprehension(self): |
| self.check_ast_roundtrip("{x for x in range(5)}") |
| |
| def test_dict_comprehension(self): |
| self.check_ast_roundtrip("{x: x*x for x in range(10)}") |
| |
| def test_class_decorators(self): |
| self.check_ast_roundtrip(class_decorator) |
| |
| def test_class_definition(self): |
| self.check_ast_roundtrip("class A(metaclass=type, *[], **{}): pass") |
| |
| def test_elifs(self): |
| self.check_ast_roundtrip(elif1) |
| self.check_ast_roundtrip(elif2) |
| |
| def test_try_except_finally(self): |
| self.check_ast_roundtrip(try_except_finally) |
| |
| def test_starred_assignment(self): |
| self.check_ast_roundtrip("a, *b, c = seq") |
| self.check_ast_roundtrip("a, (*b, c) = seq") |
| self.check_ast_roundtrip("a, *b[0], c = seq") |
| self.check_ast_roundtrip("a, *(b, c) = seq") |
| |
| def test_with_simple(self): |
| self.check_ast_roundtrip(with_simple) |
| |
| def test_with_as(self): |
| self.check_ast_roundtrip(with_as) |
| |
| def test_with_two_items(self): |
| self.check_ast_roundtrip(with_two_items) |
| |
| def test_dict_unpacking_in_dict(self): |
| # See issue 26489 |
| self.check_ast_roundtrip(r"""{**{'y': 2}, 'x': 1}""") |
| self.check_ast_roundtrip(r"""{**{'y': 2}, **{'x': 1}}""") |
| |
| def test_slices(self): |
| self.check_ast_roundtrip("a[i]") |
| self.check_ast_roundtrip("a[i,]") |
| self.check_ast_roundtrip("a[i, j]") |
| self.check_ast_roundtrip("a[(*a,)]") |
| self.check_ast_roundtrip("a[(a:=b)]") |
| self.check_ast_roundtrip("a[(a:=b,c)]") |
| self.check_ast_roundtrip("a[()]") |
| self.check_ast_roundtrip("a[i:j]") |
| self.check_ast_roundtrip("a[:j]") |
| self.check_ast_roundtrip("a[i:]") |
| self.check_ast_roundtrip("a[i:j:k]") |
| self.check_ast_roundtrip("a[:j:k]") |
| self.check_ast_roundtrip("a[i::k]") |
| self.check_ast_roundtrip("a[i:j,]") |
| self.check_ast_roundtrip("a[i:j, k]") |
| |
| def test_invalid_raise(self): |
| self.check_invalid(ast.Raise(exc=None, cause=ast.Name(id="X"))) |
| |
| def test_invalid_fstring_constant(self): |
| self.check_invalid(ast.JoinedStr(values=[ast.Constant(value=100)])) |
| |
| def test_invalid_fstring_conversion(self): |
| self.check_invalid( |
| ast.FormattedValue( |
| value=ast.Constant(value="a", kind=None), |
| conversion=ord("Y"), # random character |
| format_spec=None, |
| ) |
| ) |
| |
| def test_invalid_fstring_backslash(self): |
| self.check_invalid(ast.FormattedValue(value=ast.Constant(value="\\\\"))) |
| |
| def test_invalid_yield_from(self): |
| self.check_invalid(ast.YieldFrom(value=None)) |
| |
| def test_docstrings(self): |
| docstrings = ( |
| 'this ends with double quote"', |
| 'this includes a """triple quote"""', |
| '\r', |
| '\\r', |
| '\t', |
| '\\t', |
| '\n', |
| '\\n', |
| '\r\\r\t\\t\n\\n', |
| '""">>> content = \"\"\"blabla\"\"\" <<<"""', |
| r'foo\n\x00', |
| "' \\'\\'\\'\"\"\" \"\"\\'\\' \\'", |
| '🐍⛎𩸽üéş^\\\\X\\\\BB\N{LONG RIGHTWARDS SQUIGGLE ARROW}' |
| ) |
| for docstring in docstrings: |
| # check as Module docstrings for easy testing |
| self.check_ast_roundtrip(f"'''{docstring}'''") |
| |
| def test_constant_tuples(self): |
| self.check_src_roundtrip(ast.Constant(value=(1,), kind=None), "(1,)") |
| self.check_src_roundtrip( |
| ast.Constant(value=(1, 2, 3), kind=None), "(1, 2, 3)" |
| ) |
| |
| def test_function_type(self): |
| for function_type in ( |
| "() -> int", |
| "(int, int) -> int", |
| "(Callable[complex], More[Complex(call.to_typevar())]) -> None" |
| ): |
| self.check_ast_roundtrip(function_type, mode="func_type") |
| |
| def test_type_comments(self): |
| for statement in ( |
| "a = 5 # type:", |
| "a = 5 # type: int", |
| "a = 5 # type: int and more", |
| "def x(): # type: () -> None\n\tpass", |
| "def x(y): # type: (int) -> None and more\n\tpass", |
| "async def x(): # type: () -> None\n\tpass", |
| "async def x(y): # type: (int) -> None and more\n\tpass", |
| "for x in y: # type: int\n\tpass", |
| "async for x in y: # type: int\n\tpass", |
| "with x(): # type: int\n\tpass", |
| "async with x(): # type: int\n\tpass" |
| ): |
| self.check_ast_roundtrip(statement, type_comments=True) |
| |
| def test_type_ignore(self): |
| for statement in ( |
| "a = 5 # type: ignore", |
| "a = 5 # type: ignore and more", |
| "def x(): # type: ignore\n\tpass", |
| "def x(y): # type: ignore and more\n\tpass", |
| "async def x(): # type: ignore\n\tpass", |
| "async def x(y): # type: ignore and more\n\tpass", |
| "for x in y: # type: ignore\n\tpass", |
| "async for x in y: # type: ignore\n\tpass", |
| "with x(): # type: ignore\n\tpass", |
| "async with x(): # type: ignore\n\tpass" |
| ): |
| self.check_ast_roundtrip(statement, type_comments=True) |
| |
| |
| class CosmeticTestCase(ASTTestCase): |
| """Test if there are cosmetic issues caused by unnecessary additions""" |
| |
| def test_simple_expressions_parens(self): |
| self.check_src_roundtrip("(a := b)") |
| self.check_src_roundtrip("await x") |
| self.check_src_roundtrip("x if x else y") |
| self.check_src_roundtrip("lambda x: x") |
| self.check_src_roundtrip("1 + 1") |
| self.check_src_roundtrip("1 + 2 / 3") |
| self.check_src_roundtrip("(1 + 2) / 3") |
| self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2)") |
| self.check_src_roundtrip("(1 + 2) * 3 + 4 * (5 + 2) ** 2") |
| self.check_src_roundtrip("~x") |
| self.check_src_roundtrip("x and y") |
| self.check_src_roundtrip("x and y and z") |
| self.check_src_roundtrip("x and (y and x)") |
| self.check_src_roundtrip("(x and y) and z") |
| self.check_src_roundtrip("(x ** y) ** z ** q") |
| self.check_src_roundtrip("x >> y") |
| self.check_src_roundtrip("x << y") |
| self.check_src_roundtrip("x >> y and x >> z") |
| self.check_src_roundtrip("x + y - z * q ^ t ** k") |
| self.check_src_roundtrip("P * V if P and V else n * R * T") |
| self.check_src_roundtrip("lambda P, V, n: P * V == n * R * T") |
| self.check_src_roundtrip("flag & (other | foo)") |
| self.check_src_roundtrip("not x == y") |
| self.check_src_roundtrip("x == (not y)") |
| self.check_src_roundtrip("yield x") |
| self.check_src_roundtrip("yield from x") |
| self.check_src_roundtrip("call((yield x))") |
| self.check_src_roundtrip("return x + (yield x)") |
| |
| def test_class_bases_and_keywords(self): |
| self.check_src_roundtrip("class X:\n pass") |
| self.check_src_roundtrip("class X(A):\n pass") |
| self.check_src_roundtrip("class X(A, B, C, D):\n pass") |
| self.check_src_roundtrip("class X(x=y):\n pass") |
| self.check_src_roundtrip("class X(metaclass=z):\n pass") |
| self.check_src_roundtrip("class X(x=y, z=d):\n pass") |
| self.check_src_roundtrip("class X(A, x=y):\n pass") |
| self.check_src_roundtrip("class X(A, **kw):\n pass") |
| self.check_src_roundtrip("class X(*args):\n pass") |
| self.check_src_roundtrip("class X(*args, **kwargs):\n pass") |
| |
| def test_fstrings(self): |
| self.check_src_roundtrip('''f\'\'\'-{f"""*{f"+{f'.{x}.'}+"}*"""}-\'\'\'''') |
| self.check_src_roundtrip('''f"\\u2028{'x'}"''') |
| self.check_src_roundtrip(r"f'{x}\n'") |
| self.check_src_roundtrip('''f''\'{"""\n"""}\\n''\'''') |
| self.check_src_roundtrip('''f''\'{f"""{x}\n"""}\\n''\'''') |
| |
| def test_docstrings(self): |
| docstrings = ( |
| '"""simple doc string"""', |
| '''"""A more complex one |
| with some newlines"""''', |
| '''"""Foo bar baz |
| |
| empty newline"""''', |
| '"""With some \t"""', |
| '"""Foo "bar" baz """', |
| '"""\\r"""', |
| '""""""', |
| '"""\'\'\'"""', |
| '"""\'\'\'\'\'\'"""', |
| '"""🐍⛎𩸽üéş^\\\\X\\\\BB⟿"""', |
| '"""end in single \'quote\'"""', |
| "'''end in double \"quote\"'''", |
| '"""almost end in double "quote"."""', |
| ) |
| |
| for prefix in docstring_prefixes: |
| for docstring in docstrings: |
| self.check_src_roundtrip(f"{prefix}{docstring}") |
| |
| def test_docstrings_negative_cases(self): |
| # Test some cases that involve strings in the children of the |
| # first node but aren't docstrings to make sure we don't have |
| # False positives. |
| docstrings_negative = ( |
| 'a = """false"""', |
| '"""false""" + """unless its optimized"""', |
| '1 + 1\n"""false"""', |
| 'f"""no, top level but f-fstring"""' |
| ) |
| for prefix in docstring_prefixes: |
| for negative in docstrings_negative: |
| # this cases should be result with single quote |
| # rather then triple quoted docstring |
| src = f"{prefix}{negative}" |
| self.check_ast_roundtrip(src) |
| self.check_src_dont_roundtrip(src) |
| |
| def test_unary_op_factor(self): |
| for prefix in ("+", "-", "~"): |
| self.check_src_roundtrip(f"{prefix}1") |
| for prefix in ("not",): |
| self.check_src_roundtrip(f"{prefix} 1") |
| |
| def test_slices(self): |
| self.check_src_roundtrip("a[1]") |
| self.check_src_roundtrip("a[1, 2]") |
| self.check_src_roundtrip("a[(1, *a)]") |
| |
| class DirectoryTestCase(ASTTestCase): |
| """Test roundtrip behaviour on all files in Lib and Lib/test.""" |
| |
| lib_dir = pathlib.Path(__file__).parent / ".." |
| test_directories = (lib_dir, lib_dir / "test") |
| run_always_files = {"test_grammar.py", "test_syntax.py", "test_compile.py", |
| "test_ast.py", "test_asdl_parser.py", "test_fstring.py", |
| "test_patma.py"} |
| |
| _files_to_test = None |
| |
| @classmethod |
| def files_to_test(cls): |
| |
| if cls._files_to_test is not None: |
| return cls._files_to_test |
| |
| items = [ |
| item.resolve() |
| for directory in cls.test_directories |
| for item in directory.glob("*.py") |
| if not item.name.startswith("bad") |
| ] |
| |
| # Test limited subset of files unless the 'cpu' resource is specified. |
| if not test.support.is_resource_enabled("cpu"): |
| |
| tests_to_run_always = {item for item in items if |
| item.name in cls.run_always_files} |
| |
| items = set(random.sample(items, 10)) |
| |
| # Make sure that at least tests that heavily use grammar features are |
| # always considered in order to reduce the chance of missing something. |
| items = list(items | tests_to_run_always) |
| |
| # bpo-31174: Store the names sample to always test the same files. |
| # It prevents false alarms when hunting reference leaks. |
| cls._files_to_test = items |
| |
| return items |
| |
| def test_files(self): |
| for item in self.files_to_test(): |
| if test.support.verbose: |
| print(f"Testing {item.absolute()}") |
| |
| with self.subTest(filename=item): |
| source = read_pyfile(item) |
| self.check_ast_roundtrip(source) |
| |
| |
| if __name__ == "__main__": |
| unittest.main() |
