From 91319e7da3a05757b478a692b8ac70af40cbfb54 Mon Sep 17 00:00:00 2001 From: rechen Date: Thu, 30 Apr 2020 15:13:48 -0700 Subject: [PATCH 1/6] Fix a crash in typing.overload inference. output.py was using the signature from a function's implementation to generate pytd output for its overloads, leading to crashes when the overload's signature differed significantly from the implementation. Fixed by iterating through the overloads in output.py rather than in InterpreterFunction.get_call_combinations, since moving the loop outward means that the entire function is replaced for each overload. This change lets us stop saving/passing around the signature in various places, since we can get it directly from the function again. I also fixed a few places where I noticed InterpreterFunction.call using self.signature rather than the signature from the current overload. PiperOrigin-RevId: 309308762 --- pytype/abstract.py | 16 +++-- pytype/output.py | 101 ++++++++++++++++-------------- pytype/tests/py3/test_overload.py | 40 ++++++++++++ 3 files changed, 102 insertions(+), 55 deletions(-) diff --git a/pytype/abstract.py b/pytype/abstract.py index 847fd6838..4ae3a4ee5 100644 --- a/pytype/abstract.py +++ b/pytype/abstract.py @@ -3119,7 +3119,7 @@ def call(self, node, func, args, new_locals=False, alias_map=None): for name in callargs: if (name in annotations and (not self.is_attribute_of_class or self.argcount(node) == 0 or - name != self.signature.param_names[0])): + name != sig.param_names[0])): extra_key = (self.get_first_opcode(), name) node, callargs[name] = self.vm.init_class( node, annotations[name], extra_key=extra_key) @@ -3138,8 +3138,7 @@ def call(self, node, func, args, new_locals=False, alias_map=None): # as incomplete. self._set_callself_maybe_missing_members() return node, self.vm.new_unsolvable(node) - self_var = (self.signature.param_names and - callargs.get(self.signature.param_names[0])) + self_var = sig.param_names and callargs.get(sig.param_names[0]) caller_is_abstract = abstract_utils.check_classes( self_var, lambda cls: cls.is_abstract) caller_is_protocol = abstract_utils.check_classes( @@ -3181,7 +3180,7 @@ def call(self, node, func, args, new_locals=False, alias_map=None): ret = old_ret.AssignToNewVariable(node) if self._store_call_records: # Even if the call is cached, we might not have been recording it. - self._call_records.append((sig, callargs, ret, node)) + self._call_records.append((callargs, ret, node)) return node, ret if self.code.has_generator(): generator = Generator(frame, self.vm) @@ -3212,7 +3211,7 @@ def call(self, node, func, args, new_locals=False, alias_map=None): node_after_call = abstract_utils.apply_mutations(node_after_call, mutations) self._call_cache[callkey] = ret, self.vm.remaining_depth() if self._store_call_records or self.vm.store_all_calls: - self._call_records.append((sig, callargs, ret, node_after_call)) + self._call_records.append((callargs, ret, node_after_call)) self.last_frame = frame return node_after_call, ret @@ -3220,7 +3219,7 @@ def get_call_combinations(self, node): """Get this function's call records.""" all_combinations = [] signature_data = set() - for sig, callargs, ret, node_after_call in self._call_records: + for callargs, ret, node_after_call in self._call_records: try: combinations = cfg_utils.variable_product_dict(callargs) except cfg_utils.TooComplexError: @@ -3241,15 +3240,14 @@ def get_call_combinations(self, node): node_after_call.HasCombination(values)): signature_data.add(data) all_combinations.append( - (sig, node_after_call, combination, return_value)) + (node_after_call, combination, return_value)) if not all_combinations: # Fallback: Generate signatures only from the definition of the # method, not the way it's being used. param_binding = self.vm.convert.unsolvable.to_binding(node) params = collections.defaultdict(lambda: param_binding) ret = self.vm.convert.unsolvable.to_binding(node) - for f in self.signature_functions(): - all_combinations.append((f.signature, node, params, ret)) + all_combinations.append((node, params, ret)) return all_combinations def get_positional_names(self): diff --git a/pytype/output.py b/pytype/output.py index de5e75021..6f4c65e09 100644 --- a/pytype/output.py +++ b/pytype/output.py @@ -365,14 +365,13 @@ def uninitialized_annotations_to_instance_types(self, node, annots, members): yield name, pytd_utils.JoinTypes( value.get_instance_type(node) for value in annots[name].data) - def _function_call_to_return_type( - self, sig, node, v, seen_return, num_returns): + def _function_call_to_return_type(self, node, v, seen_return, num_returns): """Get a function call's pytd return type.""" - if sig.has_return_annotation: + if v.signature.has_return_annotation: if v.is_coroutine(): ret = abstract.Coroutine.make(self.vm, v, node).to_type(node) else: - ret = sig.annotations["return"].get_instance_type(node) + ret = v.signature.annotations["return"].get_instance_type(node) else: ret = seen_return.data.to_type(node) if isinstance(ret, pytd.NothingType) and num_returns == 1: @@ -382,49 +381,59 @@ def _function_call_to_return_type( assert isinstance(seen_return.data, typing_overlay.NoReturn) return ret - def _function_to_def(self, node, v, function_name): - """Convert an InterpreterFunction to a PyTD definition.""" - signatures = [] - combinations = v.get_call_combinations(node) - for sig, node_after, combination, return_value in combinations: - params = [] - for i, (name, kwonly, optional) in enumerate(v.get_parameters()): - if i < v.nonstararg_count and name in sig.annotations: - t = sig.annotations[name].get_instance_type(node_after) - else: - t = combination[name].data.to_type(node_after) - # Python uses ".0" etc. for the names of parameters that are tuples, - # like e.g. in: "def f((x, y), z)". - params.append( - pytd.Parameter(name.replace(".", "_"), t, kwonly, optional, None)) - ret = self._function_call_to_return_type( - sig, node_after, v, return_value, len(combinations)) - if v.has_varargs(): - if sig.varargs_name in sig.annotations: - annot = sig.annotations[sig.varargs_name] - typ = annot.get_instance_type(node_after) - else: - typ = pytd.NamedType("__builtin__.tuple") - starargs = pytd.Parameter(sig.varargs_name, typ, False, True, None) + def _function_call_combination_to_signature( + self, func, call_combination, num_combinations): + node_after, combination, return_value = call_combination + params = [] + for i, (name, kwonly, optional) in enumerate(func.get_parameters()): + if i < func.nonstararg_count and name in func.signature.annotations: + t = func.signature.annotations[name].get_instance_type(node_after) else: - starargs = None - if v.has_kwargs(): - if sig.kwargs_name in sig.annotations: - annot = sig.annotations[sig.kwargs_name] - typ = annot.get_instance_type(node_after) - else: - typ = pytd.NamedType("__builtin__.dict") - starstarargs = pytd.Parameter(sig.kwargs_name, typ, False, True, None) + t = combination[name].data.to_type(node_after) + # Python uses ".0" etc. for the names of parameters that are tuples, + # like e.g. in: "def f((x, y), z)". + params.append( + pytd.Parameter(name.replace(".", "_"), t, kwonly, optional, None)) + ret = self._function_call_to_return_type( + node_after, func, return_value, num_combinations) + if func.has_varargs(): + if func.signature.varargs_name in func.signature.annotations: + annot = func.signature.annotations[func.signature.varargs_name] + typ = annot.get_instance_type(node_after) else: - starstarargs = None - signatures.append(pytd.Signature( - params=tuple(params), - starargs=starargs, - starstarargs=starstarargs, - return_type=ret, - exceptions=(), # TODO(kramm): record exceptions - template=())) + typ = pytd.NamedType("__builtin__.tuple") + starargs = pytd.Parameter( + func.signature.varargs_name, typ, False, True, None) + else: + starargs = None + if func.has_kwargs(): + if func.signature.kwargs_name in func.signature.annotations: + annot = func.signature.annotations[func.signature.kwargs_name] + typ = annot.get_instance_type(node_after) + else: + typ = pytd.NamedType("__builtin__.dict") + starstarargs = pytd.Parameter( + func.signature.kwargs_name, typ, False, True, None) + else: + starstarargs = None + return pytd.Signature( + params=tuple(params), + starargs=starargs, + starstarargs=starstarargs, + return_type=ret, + exceptions=(), # TODO(kramm): record exceptions + template=()) + def _function_to_def(self, node, v, function_name): + """Convert an InterpreterFunction to a PyTD definition.""" + signatures = [] + for func in v.signature_functions(): + combinations = func.get_call_combinations(node) + num_combinations = len(combinations) + signatures.extend( + self._function_call_combination_to_signature( + func, combination, num_combinations) + for combination in combinations) return pytd.Function(name=function_name, signatures=tuple(signatures), kind=pytd.METHOD, @@ -474,9 +483,9 @@ def _function_to_return_types(self, node, fvar): for val in options: if isinstance(val, abstract.InterpreterFunction): combinations = val.get_call_combinations(node) - for sig, node_after, _, return_value in combinations: + for node_after, _, return_value in combinations: types.append(self._function_call_to_return_type( - sig, node_after, val, return_value, len(combinations))) + node_after, val, return_value, len(combinations))) elif isinstance(val, abstract.PyTDFunction): types.extend(sig.pytd_sig.return_type for sig in val.signatures) else: diff --git a/pytype/tests/py3/test_overload.py b/pytype/tests/py3/test_overload.py index d2047ec2b..d7f7156c6 100644 --- a/pytype/tests/py3/test_overload.py +++ b/pytype/tests/py3/test_overload.py @@ -185,5 +185,45 @@ def f(x: int) -> int: """) self.assertErrorRegexes(errors, {"e": r"overload"}) + def test_varargs(self): + ty = self.Infer(""" + from typing import overload + @overload + def f() -> int: ... + @overload + def f(x: float, *args) -> float: ... + def f(*args): + return args[0] if args else 0 + """) + self.assertTypesMatchPytd(ty, """ + from typing import overload + @overload + def f() -> int: ... + @overload + def f(x: float, *args) -> float: ... + """) + + +class OverloadTestPy3(test_base.TargetPython3FeatureTest): + """Python 3 tests for typing.overload.""" + + def test_kwargs(self): + ty = self.Infer(""" + from typing import overload + @overload + def f() -> int: ... + @overload + def f(*, x: float = 0.0, **kwargs) -> float: ... + def f(**kwargs): + return kwargs['x'] if kwargs else 0 + """) + self.assertTypesMatchPytd(ty, """ + from typing import overload + @overload + def f() -> int: ... + @overload + def f(*, x: float = ..., **kwargs) -> float: ... + """) + test_base.main(globals(), __name__ == "__main__") From 7e8dd17dec0a10b497b16c31882d3fbf884bc9d5 Mon Sep 17 00:00:00 2001 From: rechen Date: Thu, 30 Apr 2020 16:26:48 -0700 Subject: [PATCH 2/6] Fix a "No MAKE_FUNCTION opcode" error triggered by overloads. For some reason, for overloaded functions the MAKE_FUNCTION opcode seems to be on the line of the "def" or "class" opening rather than the last significant line of the decorator. PiperOrigin-RevId: 309321787 --- pytype/directors.py | 2 +- pytype/directors_test.py | 6 +++--- pytype/vm_test.py | 34 ++++++++++++++++++++++++++-------- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/pytype/directors.py b/pytype/directors.py index 870858c6e..64721fd79 100644 --- a/pytype/directors.py +++ b/pytype/directors.py @@ -285,7 +285,7 @@ def _parse_source(self, src): open_decorator = True elif tok == tokenize.NAME: if open_decorator and token.string in ("class", "def"): - self.decorators.add(lineno - 1) + self.decorators.add(lineno) open_decorator = False if token.string == "def": last_function_definition = _FunctionDefinition.start(lineno) diff --git a/pytype/directors_test.py b/pytype/directors_test.py index 79274d072..8aac366d8 100644 --- a/pytype/directors_test.py +++ b/pytype/directors_test.py @@ -405,8 +405,8 @@ def bar(): pass """) self.assertEqual({ - 6, # real_decorator - 13 # decorator + 7, # real_decorator + 14 # decorator }, self._director._decorators) def test_stacked_decorators(self): @@ -421,7 +421,7 @@ class A: pass """) self.assertEqual({ - 7 # foo + 8 # foo }, self._director._decorators) diff --git a/pytype/vm_test.py b/pytype/vm_test.py index 2e3ff0f08..5dbcf13b5 100644 --- a/pytype/vm_test.py +++ b/pytype/vm_test.py @@ -246,22 +246,25 @@ def setUp(self): self.errorlog = errors.ErrorLog() self.vm = analyze.CallTracer(self.errorlog, self.options, self.loader) + def run_program(self, src): + return self.vm.run_program(textwrap.dedent(src), "", maximum_depth=10) + def test_type_comment_on_multiline_value(self): - self.vm.run_program(textwrap.dedent(""" + self.run_program(""" v = [ ("hello", "world", # type: should_be_ignored ) ] # type: dict - """), "", maximum_depth=10) + """) # In Python 3.7, STORE_NAME v is on the `("hello",` line. self.assertEqual({ 3 if self.python_version >= (3, 7) else 4: "dict", }, self.vm.director.type_comments) def test_type_comment_with_trailing_comma(self): - self.vm.run_program(textwrap.dedent(""" + self.run_program(""" v = [ ("hello", "world" @@ -272,7 +275,7 @@ def test_type_comment_with_trailing_comma(self): "world" ], # some comment ] # type: dict - """), "", maximum_depth=10) + """) # In Python 3.7, STORE_NAME v is on the `("hello",` line. self.assertEqual({ 3 if self.python_version >= (3, 7) else 4: "dict", @@ -280,7 +283,7 @@ def test_type_comment_with_trailing_comma(self): }, self.vm.director.type_comments) def test_decorators(self): - self.vm.run_program(textwrap.dedent(""" + self.run_program(""" class A: ''' @decorator in a docstring @@ -295,11 +298,11 @@ def f(x): def bar(): pass - """), "", maximum_depth=10) + """) self.assertEqual(self.vm.director.decorators, {6, 11}) def test_stacked_decorators(self): - self.vm.run_program(textwrap.dedent(""" + self.run_program(""" @decorator( x, y ) @@ -308,8 +311,23 @@ def test_stacked_decorators(self): class A: pass - """), "", maximum_depth=10) + """) self.assertEqual(self.vm.director.decorators, {6}) + def test_overload(self): + self.run_program(""" + from typing import overload + + @overload + def f() -> int: ... + + @overload + def f(x: str) -> str: ... + + def f(x=None): + return 0 if x is None else x + """) + self.assertEqual(self.vm.director.decorators, {5, 8}) + test_base.main(globals(), __name__ == "__main__") From 3ddef87fb5bc912280d9b3b0d3721b824ff9e598 Mon Sep 17 00:00:00 2001 From: mdemello Date: Thu, 30 Apr 2020 17:12:25 -0700 Subject: [PATCH 3/6] Remove target python 3.4 support Also removes a bunch of trivial checks from opcodes_test - we don't need to verify every entry in a map; a few spot-checks should be enough. PiperOrigin-RevId: 309329124 --- pytype/config_test.py | 8 +- pytype/pyc/loadmarshal_test.py | 2 +- pytype/pyc/opcodes.py | 30 +- pytype/pyc/opcodes_test.py | 964 ++---------------------- pytype/pytd/builtins/2/typing.pytd | 5 +- pytype/pytd/builtins/2and3/typing.pytd | 5 +- pytype/pytd/builtins/3/__builtin__.pytd | 5 +- pytype/pytd/builtins/3/builtins.pytd | 5 +- pytype/pytd/builtins/3/typing.pytd | 5 +- pytype/pytd/pep484_test.py | 2 +- pytype/single_test.py | 2 +- pytype/utils.py | 4 +- 12 files changed, 90 insertions(+), 947 deletions(-) diff --git a/pytype/config_test.py b/pytype/config_test.py index fda9b45ac..666299157 100644 --- a/pytype/config_test.py +++ b/pytype/config_test.py @@ -142,12 +142,12 @@ def test_dependency(self): def test_subset(self): input_options = datatypes.SimpleNamespace( - pythonpath=".", python_version="3.4") + pythonpath=".", python_version="3.5") config.Postprocessor( {"python_version"}, input_options, self.output_options).process() with self.assertRaises(AttributeError): _ = self.output_options.pythonpath # not processed - self.assertTupleEqual(self.output_options.python_version, (3, 4)) + self.assertTupleEqual(self.output_options.python_version, (3, 5)) def test_error(self): input_options = datatypes.SimpleNamespace(check=True, output="test.pyi") @@ -157,12 +157,12 @@ def test_error(self): def test_inplace(self): input_options = datatypes.SimpleNamespace( - disable="import-error,attribute-error", python_version="3.4") + disable="import-error,attribute-error", python_version="3.5") config.Postprocessor( {"disable", "python_version"}, input_options).process() self.assertSequenceEqual( input_options.disable, ["import-error", "attribute-error"]) - self.assertTupleEqual(input_options.python_version, (3, 4)) + self.assertTupleEqual(input_options.python_version, (3, 5)) def test_typeshed_default(self): input_options = datatypes.SimpleNamespace( diff --git a/pytype/pyc/loadmarshal_test.py b/pytype/pyc/loadmarshal_test.py index 702b5ab02..f516a1928 100644 --- a/pytype/pyc/loadmarshal_test.py +++ b/pytype/pyc/loadmarshal_test.py @@ -112,7 +112,7 @@ def test_load_code(self): b'z\4test' # name: 'test.py' b'\6\0\0\0' # first line no: 6 b'N') # lnotab: None - code = self.load(co, python_version=(3, 4)) + code = self.load(co, python_version=(3, 5)) self.assertEqual(code.co_argcount, 1) self.assertEqual(code.co_kwonlyargcount, 2) self.assertEqual(code.co_nlocals, 3) diff --git a/pytype/pyc/opcodes.py b/pytype/pyc/opcodes.py index 24f075228..af3e93a2d 100644 --- a/pytype/pyc/opcodes.py +++ b/pytype/pyc/opcodes.py @@ -957,7 +957,7 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args 147: MAP_ADD, } -python3_mapping = { +python_3_5_mapping = { 1: POP_TOP, 2: ROT_TWO, 3: ROT_THREE, @@ -968,6 +968,8 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args 11: UNARY_NEGATIVE, 12: UNARY_NOT, 15: UNARY_INVERT, + 16: BINARY_MATRIX_MULTIPLY, + 17: INPLACE_MATRIX_MULTIPLY, 19: BINARY_POWER, 20: BINARY_MULTIPLY, 22: BINARY_MODULO, @@ -981,7 +983,6 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args 50: GET_AITER, 51: GET_ANEXT, 52: BEFORE_ASYNC_WITH, - 54: STORE_MAP, # removed in Python 3.5.2 55: INPLACE_ADD, 56: INPLACE_SUBTRACT, 57: INPLACE_MULTIPLY, @@ -995,7 +996,7 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args 66: BINARY_OR, 67: INPLACE_POWER, 68: GET_ITER, - 69: STORE_LOCALS, # removed in Python 3.4 + 69: GET_YIELD_FROM_ITER, 70: PRINT_EXPR, 71: LOAD_BUILD_CLASS, # PRINT_ITEM in Python 2 72: YIELD_FROM, # PRINT_NEWLINE in Python 2 @@ -1006,7 +1007,8 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args 78: INPLACE_XOR, 79: INPLACE_OR, 80: BREAK_LOOP, - 81: WITH_CLEANUP, + 81: WITH_CLEANUP_START, + 82: WITH_CLEANUP_FINISH, 83: RETURN_VALUE, 84: IMPORT_STAR, 86: YIELD_VALUE, @@ -1064,6 +1066,11 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args 146: SET_ADD, 147: MAP_ADD, 148: LOAD_CLASSDEREF, # not in Python 2 + 149: BUILD_LIST_UNPACK, + 150: BUILD_MAP_UNPACK, + 151: BUILD_MAP_UNPACK_WITH_CALL, + 152: BUILD_TUPLE_UNPACK, + 153: BUILD_SET_UNPACK, 154: SETUP_ASYNC_WITH, } @@ -1073,20 +1080,6 @@ def _overlay_mapping(mapping, new_entries): ret.update(new_entries) return dict((k, v) for k, v in six.iteritems(ret) if v is not None) -python_3_5_mapping = _overlay_mapping(python3_mapping, { - 16: BINARY_MATRIX_MULTIPLY, - 17: INPLACE_MATRIX_MULTIPLY, - 54: None, - 69: GET_YIELD_FROM_ITER, # STORE_LOCALS in Python 3.3 - 81: WITH_CLEANUP_START, # WITH_CLEANUP in Python 3.4 - 82: WITH_CLEANUP_FINISH, - 149: BUILD_LIST_UNPACK, - 150: BUILD_MAP_UNPACK, - 151: BUILD_MAP_UNPACK_WITH_CALL, - 152: BUILD_TUPLE_UNPACK, - 153: BUILD_SET_UNPACK, -}) - python_3_6_mapping = _overlay_mapping(python_3_5_mapping, { 85: SETUP_ANNOTATIONS, 127: STORE_ANNOTATION, # removed in Python 3.7 @@ -1275,7 +1268,6 @@ def dis(data, python_version, *args, **kwargs): assert major in (2, 3) mapping = { (2, 7): python2_mapping, - (3, 4): python3_mapping, (3, 5): python_3_5_mapping, (3, 6): python_3_6_mapping, (3, 7): python_3_7_mapping, diff --git a/pytype/pyc/opcodes_test.py b/pytype/pyc/opcodes_test.py index 583a28c35..67452970c 100644 --- a/pytype/pyc/opcodes_test.py +++ b/pytype/pyc/opcodes_test.py @@ -18,6 +18,16 @@ def assertName(self, code, name): """Assert that the first diassembled opcode has the given name.""" self.assertEqual(self.dis(code)[0].name, name) + def assertDisassembly(self, code, expected): + """Assert that an extended code sequence has the expected disassembly.""" + ops = self.dis(code) + self.assertEqual(len(ops), len(expected)) + for o, e in zip(ops, expected): + if len(e) == 1: + self.assertEqual(e, (o.name,)) + else: + self.assertEqual(e, (o.name, o.arg)) + class CommonTest(_TestBase): """Test bytecodes that are common to Python 2 and 3 using python 2.""" @@ -27,281 +37,14 @@ class CommonTest(_TestBase): def test_pop_top(self): self.assertSimple(1, 'POP_TOP') - def test_rot_two(self): - self.assertSimple(2, 'ROT_TWO') - - def test_rot_three(self): - self.assertSimple(3, 'ROT_THREE') - - def test_dup_top(self): - self.assertSimple(4, 'DUP_TOP') - - def test_nop(self): - self.assertSimple(9, 'NOP') - - def test_unary_positive(self): - self.assertSimple(10, 'UNARY_POSITIVE') - - def test_unary_negative(self): - self.assertSimple(11, 'UNARY_NEGATIVE') - - def test_unary_not(self): - self.assertSimple(12, 'UNARY_NOT') - - def test_unary_invert(self): - self.assertSimple(15, 'UNARY_INVERT') - - def test_binary_power(self): - self.assertSimple(19, 'BINARY_POWER') - - def test_binary_multiply(self): - self.assertSimple(20, 'BINARY_MULTIPLY') - - def test_binary_modulo(self): - self.assertSimple(22, 'BINARY_MODULO') - - def test_binary_add(self): - self.assertSimple(23, 'BINARY_ADD') - - def test_binary_subtract(self): - self.assertSimple(24, 'BINARY_SUBTRACT') - - def test_binary_subscr(self): - self.assertSimple(25, 'BINARY_SUBSCR') - - def test_binary_floor_divide(self): - self.assertSimple(26, 'BINARY_FLOOR_DIVIDE') - - def test_binary_true_divide(self): - self.assertSimple(27, 'BINARY_TRUE_DIVIDE') - - def test_inplace_floor_divide(self): - self.assertSimple(28, 'INPLACE_FLOOR_DIVIDE') - - def test_inplace_true_divide(self): - self.assertSimple(29, 'INPLACE_TRUE_DIVIDE') - - def test_store_map(self): - self.assertSimple(54, 'STORE_MAP') - - def test_inplace_add(self): - self.assertSimple(55, 'INPLACE_ADD') - - def test_inplace_subtract(self): - self.assertSimple(56, 'INPLACE_SUBTRACT') - - def test_inplace_multiply(self): - self.assertSimple(57, 'INPLACE_MULTIPLY') - - def test_inplace_modulo(self): - self.assertSimple(59, 'INPLACE_MODULO') - - def test_store_subscr(self): - self.assertSimple(60, 'STORE_SUBSCR') - - def test_delete_subscr(self): - self.assertSimple(61, 'DELETE_SUBSCR') - - def test_binary_lshift(self): - self.assertSimple(62, 'BINARY_LSHIFT') - - def test_binary_rshift(self): - self.assertSimple(63, 'BINARY_RSHIFT') - - def test_binary_and(self): - self.assertSimple(64, 'BINARY_AND') - - def test_binary_xor(self): - self.assertSimple(65, 'BINARY_XOR') - - def test_binary_or(self): - self.assertSimple(66, 'BINARY_OR') - - def test_inplace_power(self): - self.assertSimple(67, 'INPLACE_POWER') - - def test_get_iter(self): - self.assertSimple(68, 'GET_ITER') - - def test_print_expr(self): - self.assertSimple(70, 'PRINT_EXPR') - - def test_inplace_lshift(self): - self.assertSimple(75, 'INPLACE_LSHIFT') - - def test_inplace_rshift(self): - self.assertSimple(76, 'INPLACE_RSHIFT') - - def test_inplace_and(self): - self.assertSimple(77, 'INPLACE_AND') - - def test_inplace_xor(self): - self.assertSimple(78, 'INPLACE_XOR') - - def test_inplace_or(self): - self.assertSimple(79, 'INPLACE_OR') - - def test_break_loop(self): - self.assertSimple(80, 'BREAK_LOOP') - - def test_with_cleanup(self): - self.assertSimple(81, 'WITH_CLEANUP') - - def test_return_value(self): - self.assertSimple(83, 'RETURN_VALUE') - - def test_import_star(self): - self.assertSimple(84, 'IMPORT_STAR') - - def test_yield_value(self): - self.assertSimple(86, 'YIELD_VALUE') - - def test_pop_block(self): - self.assertSimple(87, 'POP_BLOCK') - - def test_end_finally(self): - self.assertSimple(88, 'END_FINALLY') - def test_store_name(self): self.assertName([90, 0, 0], 'STORE_NAME') - def test_delete_name(self): - self.assertName([91, 0, 0], 'DELETE_NAME') - - def test_unpack_sequence(self): - self.assertName([92, 0, 0], 'UNPACK_SEQUENCE') - def test_for_iter(self): self.assertName([93, 0, 0, 9], 'FOR_ITER') - def test_store_attr(self): - self.assertName([95, 0, 0], 'STORE_ATTR') - - def test_delete_attr(self): - self.assertName([96, 0, 0], 'DELETE_ATTR') - - def test_store_global(self): - self.assertName([97, 0, 0], 'STORE_GLOBAL') - - def test_delete_global(self): - self.assertName([98, 0, 0], 'DELETE_GLOBAL') - - def test_load_const(self): - self.assertName([100, 0, 0], 'LOAD_CONST') - - def test_load_name(self): - self.assertName([101, 0, 0], 'LOAD_NAME') - - def test_build_tuple(self): - self.assertName([102, 0, 0], 'BUILD_TUPLE') - - def test_build_list(self): - self.assertName([103, 0, 0], 'BUILD_LIST') - - def test_build_set(self): - self.assertName([104, 0, 0], 'BUILD_SET') - - def test_build_map(self): - self.assertName([105, 0, 0], 'BUILD_MAP') - - def test_load_attr(self): - self.assertName([106, 0, 0], 'LOAD_ATTR') - - def test_compare_op(self): - self.assertName([107, 0, 0], 'COMPARE_OP') - - def test_import_name(self): - self.assertName([108, 0, 0], 'IMPORT_NAME') - - def test_import_from(self): - self.assertName([109, 0, 0], 'IMPORT_FROM') - - def test_jump_forward(self): - self.assertName([110, 0, 0, 9], 'JUMP_FORWARD') - - def test_jump_if_false_or_pop(self): - self.assertName([111, 3, 0, 9], 'JUMP_IF_FALSE_OR_POP') - - def test_jump_if_true_or_pop(self): - self.assertName([112, 3, 0, 9], 'JUMP_IF_TRUE_OR_POP') - - def test_jump_absolute(self): - self.assertName([113, 3, 0, 9], 'JUMP_ABSOLUTE') - - def test_pop_jump_if_false(self): - self.assertName([114, 3, 0, 9], 'POP_JUMP_IF_FALSE') - - def test_pop_jump_if_true(self): - self.assertName([115, 3, 0, 9], 'POP_JUMP_IF_TRUE') - - def test_load_global(self): - self.assertName([116, 0, 0], 'LOAD_GLOBAL') - - def test_continue_loop(self): - self.assertName([119, 3, 0, 9], 'CONTINUE_LOOP') - - def test_setup_loop(self): - self.assertName([120, 0, 0, 9], 'SETUP_LOOP') - - def test_setup_except(self): - self.assertName([121, 0, 0, 9], 'SETUP_EXCEPT') - - def test_setup_finally(self): - self.assertName([122, 0, 0, 9], 'SETUP_FINALLY') - - def test_load_fast(self): - self.assertName([124, 0, 0], 'LOAD_FAST') - - def test_store_fast(self): - self.assertName([125, 0, 0], 'STORE_FAST') - - def test_delete_fast(self): - self.assertName([126, 0, 0], 'DELETE_FAST') - - def test_raise_varargs(self): - self.assertName([130, 0, 0], 'RAISE_VARARGS') - - def test_call_function(self): - self.assertName([131, 0, 0], 'CALL_FUNCTION') - - def test_make_function(self): - self.assertName([132, 0, 0], 'MAKE_FUNCTION') - - def test_build_slice(self): - self.assertName([133, 0, 0], 'BUILD_SLICE') - - def test_make_closure(self): - self.assertName([134, 0, 0], 'MAKE_CLOSURE') - - def test_load_closure(self): - self.assertName([135, 0, 0], 'LOAD_CLOSURE') - - def test_load_deref(self): - self.assertName([136, 0, 0], 'LOAD_DEREF') - - def test_store_deref(self): - self.assertName([137, 0, 0], 'STORE_DEREF') - - def test_call_function_var(self): - self.assertName([140, 0, 0], 'CALL_FUNCTION_VAR') - - def test_call_function_kw(self): - self.assertName([141, 0, 0], 'CALL_FUNCTION_KW') - - def test_call_function_var_kw(self): - self.assertName([142, 0, 0], 'CALL_FUNCTION_VAR_KW') - - def test_setup_with(self): - self.assertName([143, 0, 0, 9], 'SETUP_WITH') - - def test_set_add(self): - self.assertName([146, 0, 0], 'SET_ADD') - - def test_map_add(self): - self.assertName([147, 0, 0], 'MAP_ADD') - - def test_binary(self): - ops = self.dis([ + def test_extended_disassembly(self): + code = [ 0x7c, 0, 0, # 0 LOAD_FAST, arg=0, 0x7c, 0, 0, # 3 LOAD_FAST, arg=0, 0x17, # 6 BINARY_ADD, @@ -320,251 +63,34 @@ def test_binary(self): 0x01, # 31 POP_TOP, 0x64, 0, 0, # 32 LOAD_CONST, arg=0, 0x53, # 35 RETURN_VALUE - ]) - self.assertEqual(len(ops), 18) - self.assertEqual(ops[0].name, 'LOAD_FAST') - self.assertEqual(ops[0].arg, 0) - self.assertEqual(ops[1].name, 'LOAD_FAST') - self.assertEqual(ops[1].arg, 0) - self.assertEqual(ops[2].name, 'BINARY_ADD') - self.assertEqual(ops[3].name, 'POP_TOP') - self.assertEqual(ops[4].name, 'LOAD_FAST') - self.assertEqual(ops[4].arg, 0) - self.assertEqual(ops[5].name, 'LOAD_FAST') - self.assertEqual(ops[5].arg, 0) - self.assertEqual(ops[6].name, 'BINARY_MULTIPLY') - self.assertEqual(ops[7].name, 'POP_TOP') - self.assertEqual(ops[8].name, 'LOAD_FAST') - self.assertEqual(ops[8].arg, 0) - self.assertEqual(ops[9].name, 'LOAD_FAST') - self.assertEqual(ops[9].arg, 0) - self.assertEqual(ops[10].name, 'BINARY_MODULO') - self.assertEqual(ops[11].name, 'POP_TOP') - self.assertEqual(ops[12].name, 'LOAD_FAST') - self.assertEqual(ops[12].arg, 0) - self.assertEqual(ops[13].name, 'LOAD_FAST') - self.assertEqual(ops[13].arg, 0) - self.assertEqual(ops[14].name, 'BINARY_TRUE_DIVIDE') - self.assertEqual(ops[15].name, 'POP_TOP') - self.assertEqual(ops[16].name, 'LOAD_CONST') - self.assertEqual(ops[16].arg, 0) - self.assertEqual(ops[17].name, 'RETURN_VALUE') - - def test_break(self): - ops = self.dis([ - 0x78, 4, 0, # 0 SETUP_LOOP, dest=7, - 0x50, # 3 BREAK_LOOP, - 0x71, 3, 0, # 4 JUMP_ABSOLUTE, dest=3, - 0x64, 0, 0, # 7 LOAD_CONST, arg=0, - 0x53, # 10 RETURN_VALUE - ]) - self.assertEqual(len(ops), 5) - self.assertEqual(ops[0].name, 'SETUP_LOOP') - self.assertEqual(ops[0].arg, 3) - self.assertEqual(ops[0].target, ops[3]) - self.assertEqual(ops[1].name, 'BREAK_LOOP') - self.assertEqual(ops[2].name, 'JUMP_ABSOLUTE') - self.assertEqual(ops[2].arg, 1) - self.assertEqual(ops[2].target, ops[1]) - self.assertEqual(ops[3].name, 'LOAD_CONST') - self.assertEqual(ops[3].arg, 0) - self.assertEqual(ops[4].name, 'RETURN_VALUE') - - def test_call(self): - ops = self.dis([ - 0x74, 0, 0, # 0 LOAD_GLOBAL, arg=0, - 0x83, 0, 0, # 3 CALL_FUNCTION, arg=0, - 0x01, # 6 POP_TOP, - 0x64, 0, 0, # 7 LOAD_CONST, arg=0, - 0x53, # 10 RETURN_VALUE - ]) - self.assertEqual(len(ops), 5) - self.assertEqual(ops[0].name, 'LOAD_GLOBAL') - self.assertEqual(ops[0].arg, 0) - self.assertEqual(ops[1].name, 'CALL_FUNCTION') - self.assertEqual(ops[1].arg, 0) - self.assertEqual(ops[2].name, 'POP_TOP') - self.assertEqual(ops[3].name, 'LOAD_CONST') - self.assertEqual(ops[3].arg, 0) - self.assertEqual(ops[4].name, 'RETURN_VALUE') - - def test_continue(self): - ops = self.dis([ - 0x78, 6, 0, # 0 SETUP_LOOP, dest=9, - 0x71, 3, 0, # 3 JUMP_ABSOLUTE, dest=3, - 0x71, 3, 0, # 6 JUMP_ABSOLUTE, dest=3, - 0x64, 0, 0, # 9 LOAD_CONST, arg=0, - 0x53, # 12 RETURN_VALUE - ]) - self.assertEqual(len(ops), 5) - self.assertEqual(ops[0].name, 'SETUP_LOOP') - self.assertEqual(ops[0].arg, 3) - self.assertEqual(ops[0].target, ops[3]) - self.assertEqual(ops[1].name, 'JUMP_ABSOLUTE') - self.assertEqual(ops[1].arg, 1) - self.assertEqual(ops[1].target, ops[1]) - self.assertEqual(ops[2].name, 'JUMP_ABSOLUTE') - self.assertEqual(ops[2].arg, 1) - self.assertEqual(ops[2].target, ops[1]) - self.assertEqual(ops[3].name, 'LOAD_CONST') - self.assertEqual(ops[3].arg, 0) - self.assertEqual(ops[4].name, 'RETURN_VALUE') - - def test_finally(self): - ops = self.dis([ - 0x7a, 4, 0, # 0 SETUP_FINALLY, dest=7, - 0x57, # 3 POP_BLOCK, - 0x64, 0, 0, # 4 LOAD_CONST, arg=0, - 0x58, # 7 END_FINALLY, - 0x64, 0, 0, # 8 LOAD_CONST, arg=0, - 0x53, # 11 RETURN_VALUE - ]) - self.assertEqual(len(ops), 6) - self.assertEqual(ops[0].name, 'SETUP_FINALLY') - self.assertEqual(ops[0].arg, 3) - self.assertEqual(ops[0].target, ops[3]) - self.assertEqual(ops[1].name, 'POP_BLOCK') - self.assertEqual(ops[2].name, 'LOAD_CONST') - self.assertEqual(ops[2].arg, 0) - self.assertEqual(ops[3].name, 'END_FINALLY') - self.assertEqual(ops[4].name, 'LOAD_CONST') - self.assertEqual(ops[4].arg, 0) - self.assertEqual(ops[5].name, 'RETURN_VALUE') - - def test_inplace(self): - ops = self.dis([ - 0x7c, 0, 0, # 0 LOAD_FAST, arg=0, - 0x7c, 0, 0, # 3 LOAD_FAST, arg=0, - 0x4b, # 6 INPLACE_LSHIFT, - 0x7d, 0, 0, # 7 STORE_FAST, arg=0, - 0x7c, 0, 0, # 10 LOAD_FAST, arg=0, - 0x7c, 0, 0, # 13 LOAD_FAST, arg=0, - 0x4c, # 16 INPLACE_RSHIFT, - 0x7d, 0, 0, # 17 STORE_FAST, arg=0, - 0x7c, 0, 0, # 20 LOAD_FAST, arg=0, - 0x7c, 0, 0, # 23 LOAD_FAST, arg=0, - 0x37, # 26 INPLACE_ADD, - 0x7d, 0, 0, # 27 STORE_FAST, arg=0, - 0x7c, 0, 0, # 30 LOAD_FAST, arg=0, - 0x7c, 0, 0, # 33 LOAD_FAST, arg=0, - 0x38, # 36 INPLACE_SUBTRACT, - 0x7d, 0, 0, # 37 STORE_FAST, arg=0, - 0x64, 0, 0, # 40 LOAD_CONST, arg=0, - 0x53, # 43 RETURN_VALUE - ]) - self.assertEqual(len(ops), 18) - self.assertEqual(ops[0].name, 'LOAD_FAST') - self.assertEqual(ops[0].arg, 0) - self.assertEqual(ops[1].name, 'LOAD_FAST') - self.assertEqual(ops[1].arg, 0) - self.assertEqual(ops[2].name, 'INPLACE_LSHIFT') - self.assertEqual(ops[3].name, 'STORE_FAST') - self.assertEqual(ops[3].arg, 0) - self.assertEqual(ops[4].name, 'LOAD_FAST') - self.assertEqual(ops[4].arg, 0) - self.assertEqual(ops[5].name, 'LOAD_FAST') - self.assertEqual(ops[5].arg, 0) - self.assertEqual(ops[6].name, 'INPLACE_RSHIFT') - self.assertEqual(ops[7].name, 'STORE_FAST') - self.assertEqual(ops[7].arg, 0) - self.assertEqual(ops[8].name, 'LOAD_FAST') - self.assertEqual(ops[8].arg, 0) - self.assertEqual(ops[9].name, 'LOAD_FAST') - self.assertEqual(ops[9].arg, 0) - self.assertEqual(ops[10].name, 'INPLACE_ADD') - self.assertEqual(ops[11].name, 'STORE_FAST') - self.assertEqual(ops[11].arg, 0) - self.assertEqual(ops[12].name, 'LOAD_FAST') - self.assertEqual(ops[12].arg, 0) - self.assertEqual(ops[13].name, 'LOAD_FAST') - self.assertEqual(ops[13].arg, 0) - self.assertEqual(ops[14].name, 'INPLACE_SUBTRACT') - self.assertEqual(ops[15].name, 'STORE_FAST') - self.assertEqual(ops[15].arg, 0) - self.assertEqual(ops[16].name, 'LOAD_CONST') - self.assertEqual(ops[16].arg, 0) - self.assertEqual(ops[17].name, 'RETURN_VALUE') - - def test_raise_one(self): - ops = self.dis([ - 0x64, 0, 0, # 0 LOAD_CONST, arg=0, - 0x82, 1, 0, # 3 RAISE_VARARGS, arg=1, - 0x64, 0, 0, # 6 LOAD_CONST, arg=0, - 0x53, # 9 RETURN_VALUE - ]) - self.assertEqual(len(ops), 4) - self.assertEqual(ops[0].name, 'LOAD_CONST') - self.assertEqual(ops[0].arg, 0) - self.assertEqual(ops[1].name, 'RAISE_VARARGS') - self.assertEqual(ops[1].arg, 1) - self.assertEqual(ops[2].name, 'LOAD_CONST') - self.assertEqual(ops[2].arg, 0) - self.assertEqual(ops[3].name, 'RETURN_VALUE') - - def test_unary(self): - ops = self.dis([ - 0x7c, 0, 0, # 0 LOAD_FAST, arg=0, - 0x0b, # 3 UNARY_NEGATIVE, - 0x01, # 4 POP_TOP, - 0x7c, 0, 0, # 5 LOAD_FAST, arg=0, - 0x0f, # 8 UNARY_INVERT, - 0x01, # 9 POP_TOP, - 0x7c, 0, 0, # 10 LOAD_FAST, arg=0, - 0x0a, # 13 UNARY_POSITIVE, - 0x01, # 14 POP_TOP, - 0x64, 0, 0, # 15 LOAD_CONST, arg=0, - 0x53, # 18 RETURN_VALUE - ]) - self.assertEqual(len(ops), 11) - self.assertEqual(ops[0].name, 'LOAD_FAST') - self.assertEqual(ops[0].arg, 0) - self.assertEqual(ops[1].name, 'UNARY_NEGATIVE') - self.assertEqual(ops[2].name, 'POP_TOP') - self.assertEqual(ops[3].name, 'LOAD_FAST') - self.assertEqual(ops[3].arg, 0) - self.assertEqual(ops[4].name, 'UNARY_INVERT') - self.assertEqual(ops[5].name, 'POP_TOP') - self.assertEqual(ops[6].name, 'LOAD_FAST') - self.assertEqual(ops[6].arg, 0) - self.assertEqual(ops[7].name, 'UNARY_POSITIVE') - self.assertEqual(ops[8].name, 'POP_TOP') - self.assertEqual(ops[9].name, 'LOAD_CONST') - self.assertEqual(ops[9].arg, 0) - self.assertEqual(ops[10].name, 'RETURN_VALUE') - - def test_with(self): - ops = self.dis([ - 0x64, 0, 0, # 0 LOAD_CONST, arg=0, - 0x8f, 5, 0, # 3 SETUP_WITH, dest=11, - 0x01, # 6 POP_TOP, - 0x57, # 7 POP_BLOCK, - 0x64, 0, 0, # 8 LOAD_CONST, arg=0, - 0x51, # 11 WITH_CLEANUP, - 0x58, # 12 END_FINALLY, - 0x64, 0, 0, # 13 LOAD_CONST, arg=0, - 0x53, # 16 RETURN_VALUE - ]) - self.assertEqual(len(ops), 9) - self.assertEqual(ops[0].name, 'LOAD_CONST') - self.assertEqual(ops[0].arg, 0) - self.assertEqual(ops[1].name, 'SETUP_WITH') - self.assertEqual(ops[1].arg, 5) - self.assertEqual(ops[1].target, ops[5]) - self.assertEqual(ops[2].name, 'POP_TOP') - self.assertEqual(ops[3].name, 'POP_BLOCK') - self.assertEqual(ops[4].name, 'LOAD_CONST') - self.assertEqual(ops[4].arg, 0) - self.assertEqual(ops[5].name, 'WITH_CLEANUP') - self.assertEqual(ops[6].name, 'END_FINALLY') - self.assertEqual(ops[7].name, 'LOAD_CONST') - self.assertEqual(ops[7].arg, 0) - self.assertEqual(ops[8].name, 'RETURN_VALUE') + ] + expected = [ + ('LOAD_FAST', 0), + ('LOAD_FAST', 0), + ('BINARY_ADD',), + ('POP_TOP',), + ('LOAD_FAST', 0), + ('LOAD_FAST', 0), + ('BINARY_MULTIPLY',), + ('POP_TOP',), + ('LOAD_FAST', 0), + ('LOAD_FAST', 0), + ('BINARY_MODULO',), + ('POP_TOP',), + ('LOAD_FAST', 0), + ('LOAD_FAST', 0), + ('BINARY_TRUE_DIVIDE',), + ('POP_TOP',), + ('LOAD_CONST', 0), + ('RETURN_VALUE',), + ] + self.assertDisassembly(code, expected) class CommonUnder3Test(CommonTest): - """Test the common bytecodes using Python 3.4.""" + """Test the common bytecodes using Python 3.5.""" - python_version = (3, 4, 0) + python_version = (3, 5, 0) class Python2Test(_TestBase): @@ -575,196 +101,29 @@ class Python2Test(_TestBase): def test_stop_code(self): self.assertSimple(0, 'STOP_CODE') - def test_rot_four(self): - self.assertSimple(5, 'ROT_FOUR') - - def test_unary_convert(self): - self.assertSimple(13, 'UNARY_CONVERT') - - def test_binary_divide(self): - self.assertSimple(21, 'BINARY_DIVIDE') - - def test_slice_0(self): - self.assertSimple(30, 'SLICE_0') - - def test_slice_1(self): - self.assertSimple(31, 'SLICE_1') - - def test_slice_2(self): - self.assertSimple(32, 'SLICE_2') - - def test_slice_3(self): - self.assertSimple(33, 'SLICE_3') - - def test_store_slice_0(self): - self.assertSimple(40, 'STORE_SLICE_0') - - def test_store_slice_1(self): - self.assertSimple(41, 'STORE_SLICE_1') - - def test_store_slice_2(self): - self.assertSimple(42, 'STORE_SLICE_2') - - def test_store_slice_3(self): - self.assertSimple(43, 'STORE_SLICE_3') - - def test_delete_slice_0(self): - self.assertSimple(50, 'DELETE_SLICE_0') - - def test_delete_slice_1(self): - self.assertSimple(51, 'DELETE_SLICE_1') - - def test_delete_slice_2(self): - self.assertSimple(52, 'DELETE_SLICE_2') - - def test_delete_slice_3(self): - self.assertSimple(53, 'DELETE_SLICE_3') - - def test_inplace_divide(self): - self.assertSimple(58, 'INPLACE_DIVIDE') - - def test_print_item(self): - self.assertSimple(71, 'PRINT_ITEM') - - def test_print_newline(self): - self.assertSimple(72, 'PRINT_NEWLINE') - - def test_print_item_to(self): - self.assertSimple(73, 'PRINT_ITEM_TO') - - def test_print_newline_to(self): - self.assertSimple(74, 'PRINT_NEWLINE_TO') - - def test_load_locals(self): - self.assertSimple(82, 'LOAD_LOCALS') - - def test_exec_stmt(self): - self.assertSimple(85, 'EXEC_STMT') - - def test_build_class(self): - self.assertSimple(89, 'BUILD_CLASS') - - def test_list_append(self): - self.assertName([94, 0, 0], 'LIST_APPEND') + def test_store_map(self): + self.assertSimple(54, 'STORE_MAP') def test_dup_topx(self): self.assertName([99, 0, 0], 'DUP_TOPX') - def test_except(self): - ops = self.dis([ - 0x79, 4, 0, # 0 SETUP_EXCEPT, dest=7, - 0x57, # 3 POP_BLOCK, - 0x6e, 7, 0, # 4 JUMP_FORWARD, dest=14, - 0x01, # 7 POP_TOP, - 0x01, # 8 POP_TOP, - 0x01, # 9 POP_TOP, - 0x6e, 1, 0, # 10 JUMP_FORWARD, dest=14, - 0x58, # 13 END_FINALLY, - 0x64, 0, 0, # 14 LOAD_CONST, arg=0, - 0x53, # 17 RETURN_VALUE - ]) - self.assertEqual(len(ops), 10) - self.assertEqual(ops[0].name, 'SETUP_EXCEPT') - self.assertEqual(ops[0].arg, 3) - self.assertEqual(ops[0].target, ops[3]) - self.assertEqual(ops[1].name, 'POP_BLOCK') - self.assertEqual(ops[2].name, 'JUMP_FORWARD') - self.assertEqual(ops[2].arg, 8) - self.assertEqual(ops[2].target, ops[8]) - self.assertEqual(ops[3].name, 'POP_TOP') - self.assertEqual(ops[4].name, 'POP_TOP') - self.assertEqual(ops[5].name, 'POP_TOP') - self.assertEqual(ops[6].name, 'JUMP_FORWARD') - self.assertEqual(ops[6].arg, 8) - self.assertEqual(ops[6].target, ops[8]) - self.assertEqual(ops[7].name, 'END_FINALLY') - self.assertEqual(ops[8].name, 'LOAD_CONST') - self.assertEqual(ops[8].arg, 0) - self.assertEqual(ops[9].name, 'RETURN_VALUE') - - def test_list(self): - ops = self.dis([ - 0x67, 0, 0, # 0 BUILD_LIST, arg=0, - 0x7c, 0, 0, # 3 LOAD_FAST, arg=0, - 0x44, # 6 GET_ITER, - 0x5d, 12, 0, # 7 FOR_ITER, dest=22, - 0x7d, 1, 0, # 10 STORE_FAST, arg=1, - 0x7c, 1, 0, # 13 LOAD_FAST, arg=1, - 0x5e, 2, 0, # 16 LIST_APPEND, arg=2, - 0x71, 7, 0, # 19 JUMP_ABSOLUTE, dest=7, - 0x01, # 22 POP_TOP, - 0x64, 0, 0, # 23 LOAD_CONST, arg=0, - 0x53, # 26 RETURN_VALUE - ]) - self.assertEqual(len(ops), 11) - self.assertEqual(ops[0].name, 'BUILD_LIST') - self.assertEqual(ops[0].arg, 0) - self.assertEqual(ops[1].name, 'LOAD_FAST') - self.assertEqual(ops[1].arg, 0) - self.assertEqual(ops[2].name, 'GET_ITER') - self.assertEqual(ops[3].name, 'FOR_ITER') - self.assertEqual(ops[3].arg, 8) - self.assertEqual(ops[3].target, ops[8]) - self.assertEqual(ops[4].name, 'STORE_FAST') - self.assertEqual(ops[4].arg, 1) - self.assertEqual(ops[5].name, 'LOAD_FAST') - self.assertEqual(ops[5].arg, 1) - self.assertEqual(ops[6].name, 'LIST_APPEND') - self.assertEqual(ops[6].arg, 2) - self.assertEqual(ops[7].name, 'JUMP_ABSOLUTE') - self.assertEqual(ops[7].arg, 3) - self.assertEqual(ops[7].target, ops[3]) - self.assertEqual(ops[8].name, 'POP_TOP') - self.assertEqual(ops[9].name, 'LOAD_CONST') - self.assertEqual(ops[9].arg, 0) - self.assertEqual(ops[10].name, 'RETURN_VALUE') - - def test_loop(self): - ops = self.dis([ - 0x78, 10, 0, # 0 SETUP_LOOP, dest=13, - 0x74, 0, 0, # 3 LOAD_GLOBAL, arg=0, - 0x72, 12, 0, # 6 POP_JUMP_IF_FALSE, dest=12, - 0x71, 3, 0, # 9 JUMP_ABSOLUTE, dest=3, - 0x57, # 12 POP_BLOCK, - 0x64, 0, 0, # 13 LOAD_CONST, arg=0, - 0x53, # 16 RETURN_VALUE - ]) - self.assertEqual(len(ops), 7) - self.assertEqual(ops[0].name, 'SETUP_LOOP') - self.assertEqual(ops[0].arg, 5) - self.assertEqual(ops[0].target, ops[5]) - self.assertEqual(ops[1].name, 'LOAD_GLOBAL') - self.assertEqual(ops[1].arg, 0) - self.assertEqual(ops[2].name, 'POP_JUMP_IF_FALSE') - self.assertEqual(ops[2].arg, 4) - self.assertEqual(ops[2].target, ops[4]) - self.assertEqual(ops[3].name, 'JUMP_ABSOLUTE') - self.assertEqual(ops[3].arg, 1) - self.assertEqual(ops[3].target, ops[1]) - self.assertEqual(ops[4].name, 'POP_BLOCK') - self.assertEqual(ops[5].name, 'LOAD_CONST') - self.assertEqual(ops[5].arg, 0) - self.assertEqual(ops[6].name, 'RETURN_VALUE') - def test_extended_arg(self): - ops = self.dis([ + code = [ 0x91, 1, 0, # 0 EXTENDED_ARG, arg=1, 0x64, 2, 0, # 3 LOAD_CONST, arg=2 0x53, # 6 RETURN_VALUE - ]) - self.assertEqual(len(ops), 2) - self.assertEqual(ops[0].name, 'LOAD_CONST') - self.assertEqual(ops[0].arg, 0x10002) - self.assertEqual(ops[1].name, 'RETURN_VALUE') - + ] + expected = [ + ('LOAD_CONST', 0x10002), + ('RETURN_VALUE',) + ] + self.assertDisassembly(code, expected) -class Python34Test(_TestBase): - """Test bytecodes specific to Python 3.4.""" - python_version = (3, 4, 0) +class Python35Test(_TestBase): + """Test bytecodes specific to Python 3.5.""" - def test_dup_top_two(self): - self.assertSimple(5, 'DUP_TOP_TWO') + python_version = (3, 5, 2) def test_load_build_class(self): self.assertSimple(71, 'LOAD_BUILD_CLASS') @@ -772,204 +131,25 @@ def test_load_build_class(self): def test_yield_from(self): self.assertSimple(72, 'YIELD_FROM') - def test_pop_except(self): - self.assertSimple(89, 'POP_EXCEPT') + def test_with_cleanup(self): + self.assertSimple(81, 'WITH_CLEANUP_START') def test_unpack_ex(self): self.assertName([94, 0, 0], 'UNPACK_EX') - def test_delete_deref(self): - self.assertName([138, 0, 0], 'DELETE_DEREF') - - def test_list_append(self): - self.assertName([145, 0, 0], 'LIST_APPEND') - - def test_load_classderef(self): - self.assertName([148, 0, 0], 'LOAD_CLASSDEREF') - - def test_except(self): - ops = self.dis([ - 0x79, 4, 0, # 0 SETUP_EXCEPT, dest=7, - 0x57, # 3 POP_BLOCK, - 0x6e, 8, 0, # 4 JUMP_FORWARD, dest=15, - 0x01, # 7 POP_TOP, - 0x01, # 8 POP_TOP, - 0x01, # 9 POP_TOP, - 0x59, # 10 POP_EXCEPT, - 0x6e, 1, 0, # 11 JUMP_FORWARD, dest=15, - 0x58, # 14 END_FINALLY, - 0x64, 0, 0, # 15 LOAD_CONST, arg=0, - 0x53, # 18 RETURN_VALUE - ]) - self.assertEqual(len(ops), 11) - self.assertEqual(ops[0].name, 'SETUP_EXCEPT') - self.assertEqual(ops[0].arg, 3) - self.assertEqual(ops[0].target, ops[3]) - self.assertEqual(ops[1].name, 'POP_BLOCK') - self.assertEqual(ops[2].name, 'JUMP_FORWARD') - self.assertEqual(ops[2].arg, 9) - self.assertEqual(ops[2].target, ops[9]) - self.assertEqual(ops[3].name, 'POP_TOP') - self.assertEqual(ops[4].name, 'POP_TOP') - self.assertEqual(ops[5].name, 'POP_TOP') - self.assertEqual(ops[6].name, 'POP_EXCEPT') - self.assertEqual(ops[7].name, 'JUMP_FORWARD') - self.assertEqual(ops[7].arg, 9) - self.assertEqual(ops[7].target, ops[9]) - self.assertEqual(ops[8].name, 'END_FINALLY') - self.assertEqual(ops[9].name, 'LOAD_CONST') - self.assertEqual(ops[9].arg, 0) - self.assertEqual(ops[10].name, 'RETURN_VALUE') - - def test_list(self): - ops = self.dis([ - 0x64, 1, 0, # 0 LOAD_CONST, arg=1, - 0x64, 2, 0, # 3 LOAD_CONST, arg=2, - 0x84, 0, 0, # 6 MAKE_FUNCTION, arg=0, - 0x7c, 0, 0, # 9 LOAD_FAST, arg=0, - 0x44, # 12 GET_ITER, - 0x83, 1, 0, # 13 CALL_FUNCTION, arg=1, - 0x01, # 16 POP_TOP, - 0x64, 0, 0, # 17 LOAD_CONST, arg=0, - 0x53, # 20 RETURN_VALUE - ]) - self.assertEqual(len(ops), 9) - self.assertEqual(ops[0].name, 'LOAD_CONST') - self.assertEqual(ops[0].arg, 1) - self.assertEqual(ops[1].name, 'LOAD_CONST') - self.assertEqual(ops[1].arg, 2) - self.assertEqual(ops[2].name, 'MAKE_FUNCTION') - self.assertEqual(ops[2].arg, 0) - self.assertEqual(ops[3].name, 'LOAD_FAST') - self.assertEqual(ops[3].arg, 0) - self.assertEqual(ops[4].name, 'GET_ITER') - self.assertEqual(ops[5].name, 'CALL_FUNCTION') - self.assertEqual(ops[5].arg, 1) - self.assertEqual(ops[6].name, 'POP_TOP') - self.assertEqual(ops[7].name, 'LOAD_CONST') - self.assertEqual(ops[7].arg, 0) - self.assertEqual(ops[8].name, 'RETURN_VALUE') - - def test_loop(self): - ops = self.dis([ - 0x78, 3, 0, # 0 SETUP_LOOP, dest=6, - 0x71, 3, 0, # 3 JUMP_ABSOLUTE, dest=3, - 0x64, 0, 0, # 6 LOAD_CONST, arg=0, - 0x53, # 9 RETURN_VALUE - ]) - self.assertEqual(len(ops), 4) - self.assertEqual(ops[0].name, 'SETUP_LOOP') - self.assertEqual(ops[0].arg, 2) - self.assertEqual(ops[0].target, ops[2]) - self.assertEqual(ops[1].name, 'JUMP_ABSOLUTE') - self.assertEqual(ops[1].arg, 1) - self.assertEqual(ops[1].target, ops[1]) - self.assertEqual(ops[2].name, 'LOAD_CONST') - self.assertEqual(ops[2].arg, 0) - self.assertEqual(ops[3].name, 'RETURN_VALUE') - - def test_raise_zero(self): - ops = self.dis([ - 0x82, 0, 0, # 0 RAISE_VARARGS, arg=0, - 0x64, 0, 0, # 3 LOAD_CONST, arg=0, - 0x53, # 6 RETURN_VALUE - ]) - self.assertEqual(len(ops), 3) - self.assertEqual(ops[0].name, 'RAISE_VARARGS') - self.assertEqual(ops[0].arg, 0) - self.assertEqual(ops[1].name, 'LOAD_CONST') - self.assertEqual(ops[1].arg, 0) - self.assertEqual(ops[2].name, 'RETURN_VALUE') - - def test_raise_two(self): - ops = self.dis([ - 0x74, 0, 0, # 0 LOAD_GLOBAL, arg=0, - 0x74, 1, 0, # 3 LOAD_GLOBAL, arg=1, - 0x82, 2, 0, # 6 RAISE_VARARGS, arg=2, - 0x64, 0, 0, # 9 LOAD_CONST, arg=0, - 0x53, # 12 RETURN_VALUE - ]) - self.assertEqual(len(ops), 5) - self.assertEqual(ops[0].name, 'LOAD_GLOBAL') - self.assertEqual(ops[0].arg, 0) - self.assertEqual(ops[1].name, 'LOAD_GLOBAL') - self.assertEqual(ops[1].arg, 1) - self.assertEqual(ops[2].name, 'RAISE_VARARGS') - self.assertEqual(ops[2].arg, 2) - self.assertEqual(ops[3].name, 'LOAD_CONST') - self.assertEqual(ops[3].arg, 0) - self.assertEqual(ops[4].name, 'RETURN_VALUE') - - def test_raise_three(self): - ops = self.dis([ - 0x74, 0, 0, # 0 LOAD_GLOBAL, arg=0, - 0x74, 1, 0, # 3 LOAD_GLOBAL, arg=1, - 0x64, 1, 0, # 6 LOAD_CONST, arg=1, - 0x82, 3, 0, # 9 RAISE_VARARGS, arg=3, - 0x64, 0, 0, # 12 LOAD_CONST, arg=0, - 0x53, # 15 RETURN_VALUE - ]) - self.assertEqual(len(ops), 6) - self.assertEqual(ops[0].name, 'LOAD_GLOBAL') - self.assertEqual(ops[0].arg, 0) - self.assertEqual(ops[1].name, 'LOAD_GLOBAL') - self.assertEqual(ops[1].arg, 1) - self.assertEqual(ops[2].name, 'LOAD_CONST') - self.assertEqual(ops[2].arg, 1) - self.assertEqual(ops[3].name, 'RAISE_VARARGS') - self.assertEqual(ops[3].arg, 3) - self.assertEqual(ops[4].name, 'LOAD_CONST') - self.assertEqual(ops[4].arg, 0) - self.assertEqual(ops[5].name, 'RETURN_VALUE') - def test_extended_arg(self): # LOAD_CONST should be stored in the jump table with an address of 0, due to # the extended arg; if we don't do this we would throw an exception. - ops = self.dis([ + code = [ 0x90, 1, 0, # 0 EXTENDED_ARG, arg=1, 0x64, 2, 0, # 3 LOAD_CONST, arg=2 0x71, 0, 0 # 6 JUMP_ABSOLUTE, arg=0 - ]) - self.assertEqual(len(ops), 2) - self.assertEqual(ops[0].name, 'LOAD_CONST') - self.assertEqual(ops[0].arg, 0x10002) - self.assertEqual(ops[1].name, 'JUMP_ABSOLUTE') - - -class Python35Test(_TestBase): - """Test bytecodes specific to Python 3.5.""" - - python_version = (3, 5, 2) - - def test_binary_matrix_multiply(self): - self.assertSimple(16, 'BINARY_MATRIX_MULTIPLY') - - def test_inplace_matrix_multiply(self): - self.assertSimple(17, 'INPLACE_MATRIX_MULTIPLY') - - def test_get_yield_from_iter(self): - self.assertSimple(69, 'GET_YIELD_FROM_ITER') - - def test_with_cleanup_start(self): - self.assertSimple(81, 'WITH_CLEANUP_START') - - def test_with_cleanup_finish(self): - self.assertSimple(82, 'WITH_CLEANUP_FINISH') - - def test_build_list_unpack(self): - self.assertName([149, 0, 0], 'BUILD_LIST_UNPACK') - - def test_build_map_unpack(self): - self.assertName([150, 0, 0], 'BUILD_MAP_UNPACK') - - def test_build_map_unpack_with_call(self): - self.assertName([151, 0, 0], 'BUILD_MAP_UNPACK_WITH_CALL') - - def test_build_tuple_unpack(self): - self.assertName([152, 0, 0], 'BUILD_TUPLE_UNPACK') - - def test_build_set_unpack(self): - self.assertName([153, 0, 0], 'BUILD_SET_UNPACK') + ] + expected = [ + ('LOAD_CONST', 0x10002), + ('JUMP_ABSOLUTE',) + ] + self.assertDisassembly(code, expected) class Python36Test(_TestBase): @@ -983,34 +163,20 @@ def test_setup_annotations(self): def test_store_annotation(self): self.assertName([127, 0], 'STORE_ANNOTATION') - def test_call_function_ex(self): - self.assertName([142, 0], 'CALL_FUNCTION_EX') - - def test_format_value(self): - self.assertName([155, 0], 'FORMAT_VALUE') - - def test_build_const_key_map(self): - self.assertName([156, 0], 'BUILD_CONST_KEY_MAP') - - def test_build_string(self): - self.assertName([157, 0], 'BUILD_STRING') - - def test_build_tuple_unpack_with_call(self): - self.assertName([158, 0], 'BUILD_TUPLE_UNPACK_WITH_CALL') - def test_extended_arg(self): - """Same as the previous extended arg test, but using the wordcode format.""" + """Same as the py3.5 extended arg test, but using the wordcode format.""" # LOAD_CONST should be stored in the jump table with an address of 0, due to # the extended arg; if we don't do this we would throw an exception. - ops = self.dis([ + code = [ 0x90, 1, # 0 EXTENDED_ARG, arg=1, 0x64, 2, # 3 LOAD_CONST, arg=2 0x71, 0, # 6 JUMP_ABSOLUTE, arg=0 - ]) - self.assertEqual(len(ops), 2) - self.assertEqual(ops[0].name, 'LOAD_CONST') - self.assertEqual(ops[0].arg, 0x102) - self.assertEqual(ops[1].name, 'JUMP_ABSOLUTE') + ] + expected = [ + ('LOAD_CONST', 0x102), + ('JUMP_ABSOLUTE',) + ] + self.assertDisassembly(code, expected) class Python37Test(_TestBase): diff --git a/pytype/pytd/builtins/2/typing.pytd b/pytype/pytd/builtins/2/typing.pytd index ac6fc0654..236ee83fc 100644 --- a/pytype/pytd/builtins/2/typing.pytd +++ b/pytype/pytd/builtins/2/typing.pytd @@ -535,10 +535,7 @@ if sys.version_info >= (3, 0): ) -> ChainMap[Union[_K, _K2], Union[_V2, _V2]]: ... -# TODO(rechen): See https://github.com/google/pytype/issues/241. Due to a -# typeshed bug, Awaitable and Coroutine need to be defined in 3.4. Once this bug -# is fixed, we should define these classes only in 3.5+. -if sys.version_info >= (3, 4): +if sys.version_info >= (3, 5): class Awaitable(Generic[_T], Protocol): @abstractmethod def __await__(self) -> Generator[Any, None, _T]: ... diff --git a/pytype/pytd/builtins/2and3/typing.pytd b/pytype/pytd/builtins/2and3/typing.pytd index ac6fc0654..236ee83fc 100644 --- a/pytype/pytd/builtins/2and3/typing.pytd +++ b/pytype/pytd/builtins/2and3/typing.pytd @@ -535,10 +535,7 @@ if sys.version_info >= (3, 0): ) -> ChainMap[Union[_K, _K2], Union[_V2, _V2]]: ... -# TODO(rechen): See https://github.com/google/pytype/issues/241. Due to a -# typeshed bug, Awaitable and Coroutine need to be defined in 3.4. Once this bug -# is fixed, we should define these classes only in 3.5+. -if sys.version_info >= (3, 4): +if sys.version_info >= (3, 5): class Awaitable(Generic[_T], Protocol): @abstractmethod def __await__(self) -> Generator[Any, None, _T]: ... diff --git a/pytype/pytd/builtins/3/__builtin__.pytd b/pytype/pytd/builtins/3/__builtin__.pytd index b2a01e9df..fed4dea6f 100644 --- a/pytype/pytd/builtins/3/__builtin__.pytd +++ b/pytype/pytd/builtins/3/__builtin__.pytd @@ -820,10 +820,7 @@ class generator(Generator[_T, _T2, _V]): def send(self, value: _T2) -> _T def close(self) -> NoneType -# TODO(rechen): See https://github.com/google/pytype/issues/241. Due to a -# typeshed bug, coroutine needs to be defined in 3.4. Once this bug is fixed, we -# should define this class only in 3.5+. -if sys.version_info >= (3, 4): +if sys.version_info >= (3, 5): class coroutine(Coroutine[_T, _T2, _V]): __slots__ = [] def close(self) -> NoneType diff --git a/pytype/pytd/builtins/3/builtins.pytd b/pytype/pytd/builtins/3/builtins.pytd index b2a01e9df..fed4dea6f 100644 --- a/pytype/pytd/builtins/3/builtins.pytd +++ b/pytype/pytd/builtins/3/builtins.pytd @@ -820,10 +820,7 @@ class generator(Generator[_T, _T2, _V]): def send(self, value: _T2) -> _T def close(self) -> NoneType -# TODO(rechen): See https://github.com/google/pytype/issues/241. Due to a -# typeshed bug, coroutine needs to be defined in 3.4. Once this bug is fixed, we -# should define this class only in 3.5+. -if sys.version_info >= (3, 4): +if sys.version_info >= (3, 5): class coroutine(Coroutine[_T, _T2, _V]): __slots__ = [] def close(self) -> NoneType diff --git a/pytype/pytd/builtins/3/typing.pytd b/pytype/pytd/builtins/3/typing.pytd index ac6fc0654..236ee83fc 100644 --- a/pytype/pytd/builtins/3/typing.pytd +++ b/pytype/pytd/builtins/3/typing.pytd @@ -535,10 +535,7 @@ if sys.version_info >= (3, 0): ) -> ChainMap[Union[_K, _K2], Union[_V2, _V2]]: ... -# TODO(rechen): See https://github.com/google/pytype/issues/241. Due to a -# typeshed bug, Awaitable and Coroutine need to be defined in 3.4. Once this bug -# is fixed, we should define these classes only in 3.5+. -if sys.version_info >= (3, 4): +if sys.version_info >= (3, 5): class Awaitable(Generic[_T], Protocol): @abstractmethod def __await__(self) -> Generator[Any, None, _T]: ... diff --git a/pytype/pytd/pep484_test.py b/pytype/pytd/pep484_test.py index 979d51d06..eec203ca4 100644 --- a/pytype/pytd/pep484_test.py +++ b/pytype/pytd/pep484_test.py @@ -39,7 +39,7 @@ def test_convert_anystr(self): self.assertEqual(self.convert(t, python_version=(2, 7)), "AnyStr") t = pytd.NamedType("typing.AnyStr") - self.assertEqual(self.convert(t, python_version=(3, 4)), + self.assertEqual(self.convert(t, python_version=(3, 5)), "AnyStr") diff --git a/pytype/single_test.py b/pytype/single_test.py index 145ea839a..2e11261fb 100644 --- a/pytype/single_test.py +++ b/pytype/single_test.py @@ -364,7 +364,7 @@ def f(): pass """)) # Set up a python version mismatch - self.pytype_args["--python_version"] = "3.4" + self.pytype_args["--python_version"] = "3.5" self.pytype_args["--output-errors-csv"] = self.errors_csv self._run_pytype(self.pytype_args) self.assertOutputStateMatches(stdout=False, stderr=True, returncode=True) diff --git a/pytype/utils.py b/pytype/utils.py index a4767ce32..6d7069f55 100644 --- a/pytype/utils.py +++ b/pytype/utils.py @@ -84,13 +84,13 @@ def validate_version(python_version): elif (2, 8) <= python_version < (3, 0): raise UsageError("Python version %r is not a valid Python version." % format_version(python_version)) - elif (3, 0) <= python_version <= (3, 3): + elif (3, 0) <= python_version <= (3, 4): # These have odd __build_class__ parameters, store co_code.co_name fields # as unicode, and don't yet have the extra qualname parameter to # MAKE_FUNCTION. Jumping through these extra hoops is not worth it, given # that typing.py isn't introduced until 3.5, anyway. raise UsageError( - "Python versions 3.0 - 3.3 are not supported. Use 3.4 and higher.") + "Python versions 3.0 - 3.4 are not supported. Use 3.5 and higher.") elif python_version > (3, 7): # We have an explicit per-minor-version mapping in opcodes.py raise UsageError("Python versions > 3.7 are not yet supported.") From 3b98201cd915d5bd8c41d1c2ecec5e79e055b23c Mon Sep 17 00:00:00 2001 From: rechen Date: Fri, 1 May 2020 12:49:00 -0700 Subject: [PATCH 4/6] Improve annotated decorators' handling of classes. With this change, annotated decorators can pass through classes in the same module just like they can functions. (Decorating classes in different modules already worked correctly.) PiperOrigin-RevId: 309456542 --- pytype/abstract.py | 3 +++ pytype/tests/py3/test_decorators.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/pytype/abstract.py b/pytype/abstract.py index 4ae3a4ee5..1925936df 100644 --- a/pytype/abstract.py +++ b/pytype/abstract.py @@ -667,6 +667,9 @@ def get_class(self): # See Py_TYPE() in Include/object.h if self.cls: return self.cls + elif isinstance(self, InterpreterClass): + return ParameterizedClass( + self.vm.convert.type_type, {abstract_utils.T: self}, self.vm) elif isinstance(self, (AnnotationClass, mixin.Class)): return self.vm.convert.type_type diff --git a/pytype/tests/py3/test_decorators.py b/pytype/tests/py3/test_decorators.py index f1f55017b..0e81af388 100644 --- a/pytype/tests/py3/test_decorators.py +++ b/pytype/tests/py3/test_decorators.py @@ -38,4 +38,29 @@ def func(self, x): x.func(12) """) + def test_class_decorators(self): + ty = self.Infer(""" + from typing import Callable, TypeVar + C = TypeVar('C') + def decorator(cls: C) -> C: + return cls + def decorator_factory() -> Callable[[C], C]: + return lambda x: x + @decorator + class Foo: + pass + @decorator_factory() + class Bar: + pass + """) + self.assertTypesMatchPytd(ty, """ + from typing import Callable, TypeVar + C = TypeVar('C') + def decorator(cls: C) -> C: ... + def decorator_factory() -> Callable[[C], C]: ... + class Foo: ... + class Bar: ... + """) + + test_base.main(globals(), __name__ == "__main__") From ee90a8d63bf0141e30d3272665595098bf9adc8f Mon Sep 17 00:00:00 2001 From: rechen Date: Fri, 1 May 2020 13:07:18 -0700 Subject: [PATCH 5/6] Fix crash caused by attr.s being passed a decorated class. PiperOrigin-RevId: 309459874 --- pytype/overlays/classgen.py | 4 ++++ pytype/tests/CMakeLists.txt | 1 + pytype/tests/py3/test_attr.py | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/pytype/overlays/classgen.py b/pytype/overlays/classgen.py index 61922bdb1..05667015d 100644 --- a/pytype/overlays/classgen.py +++ b/pytype/overlays/classgen.py @@ -140,6 +140,10 @@ def get_class_locals(self, cls, allow_methods, ordering): # TODO(rechen): Once we drop Python 2 support, either use a normal dict or # replace key deletion with OrderedDict.move_to_end(). out = collections.OrderedDict() + if cls.name not in self.vm.local_ops: + # See TestAttribPy3.test_cannot_decorate in tests/py3/test_attr.py. The + # class will not be in local_ops if a previous decorator hides it. + return out for op in self.vm.local_ops[cls.name]: if is_dunder(op.name): continue diff --git a/pytype/tests/CMakeLists.txt b/pytype/tests/CMakeLists.txt index 34fd13064..22eed15a8 100644 --- a/pytype/tests/CMakeLists.txt +++ b/pytype/tests/CMakeLists.txt @@ -523,6 +523,7 @@ py_test( test_attr.py DEPS .test_base + pytype.utils ) py_test( diff --git a/pytype/tests/py3/test_attr.py b/pytype/tests/py3/test_attr.py index 0ababc98f..f251acaa7 100644 --- a/pytype/tests/py3/test_attr.py +++ b/pytype/tests/py3/test_attr.py @@ -1,5 +1,6 @@ """Tests for attrs library in attr_overlay.py.""" +from pytype import file_utils from pytype.tests import test_base @@ -104,6 +105,28 @@ class Foo(object): def __init__(self, x: int = ..., y: str = ...) -> None: ... """) + def test_cannot_decorate(self): + # Tests the attr.s decorator being passed an object it can't process. + with file_utils.Tempdir() as d: + d.create_file("foo.pyi", """ + from typing import Type + class Foo: ... + def decorate(cls: Type[Foo]) -> Type[Foo]: ... + """) + ty = self.Infer(""" + import attr + import foo + @attr.s + @foo.decorate + class Bar(foo.Foo): ... + """, pythonpath=[d.path]) + self.assertTypesMatchPytd(ty, """ + from typing import Type + attr: module + foo: module + Bar: Type[foo.Foo] + """) + class TestAttrs(test_base.TargetPython3FeatureTest): """Tests for attr.s.""" From 4ec8cbb5d237c7c5c1ac878f28bf929b19ae6975 Mon Sep 17 00:00:00 2001 From: rechen Date: Mon, 4 May 2020 18:18:28 -0700 Subject: [PATCH 6/6] Add tests for annotated decorators that can also serve as examples. Creates a suite of tests for ensuring that pytype passes functions and classes through annotated decorators as expected. We can also point users at these tests as documentation for how to annotate decorators. PiperOrigin-RevId: 309858745 --- pytype/tests/py3/CMakeLists.txt | 3 +- pytype/tests/py3/test_decorators.py | 215 ++++++++++++++++++++++++---- pytype/tests/py3/test_functions.py | 30 ---- 3 files changed, 193 insertions(+), 55 deletions(-) diff --git a/pytype/tests/py3/CMakeLists.txt b/pytype/tests/py3/CMakeLists.txt index aeb10e18a..37f92a336 100644 --- a/pytype/tests/py3/CMakeLists.txt +++ b/pytype/tests/py3/CMakeLists.txt @@ -87,6 +87,8 @@ py_test( SRCS test_decorators.py DEPS + pytype.pytd_utils + pytype.utils pytype.tests.test_base ) @@ -123,7 +125,6 @@ py_test( SRCS test_functions.py DEPS - pytype.pytd_utils pytype.tests.test_base ) diff --git a/pytype/tests/py3/test_decorators.py b/pytype/tests/py3/test_decorators.py index 0e81af388..b682d4af7 100644 --- a/pytype/tests/py3/test_decorators.py +++ b/pytype/tests/py3/test_decorators.py @@ -1,8 +1,199 @@ """Test decorators.""" +from pytype import file_utils +from pytype.pytd import pytd_utils from pytype.tests import test_base +class AnnotatedDecoratorsTest(test_base.TargetPython3BasicTest): + """A collection of tested examples of annotated decorators.""" + + def test_identity_decorator(self): + ty = self.Infer(""" + from typing import Any, Callable, TypeVar + Fn = TypeVar('Fn', bound=Callable[..., Any]) + + def decorate(fn: Fn) -> Fn: + return fn + + @decorate + def f(x: int) -> str: + return str(x) + + @decorate + class Foo: + pass + """) + self.assertTypesMatchPytd(ty, """ + from typing import Callable, TypeVar + Fn = TypeVar('Fn', bound=Callable) + def decorate(fn: Fn) -> Fn: ... + def f(x: int) -> str: ... + class Foo: ... + """) + # Prints the inferred types as a stub file and tests that the decorator + # works correctly when imported in another file. + with file_utils.Tempdir() as d: + d.create_file("foo.pyi", pytd_utils.Print(ty)) + ty = self.Infer(""" + import foo + + @foo.decorate + def f(x: str) -> int: + return int(x) + + @foo.decorate + class Bar: + pass + """, pythonpath=[d.path]) + self.assertTypesMatchPytd(ty, """ + foo: module + def f(x: str) -> int: ... + class Bar: ... + """) + + def test_decorator_factory(self): + ty = self.Infer(""" + from typing import Any, Callable, TypeVar + Fn = TypeVar('Fn', bound=Callable[..., Any]) + + def decorate(**options: Any) -> Callable[[Fn], Fn]: + def inner(fn): + return fn + return inner + + @decorate() + def f(x: int) -> str: + return str(x) + + @decorate(x=0, y=False) + def g() -> float: + return 0.0 + + @decorate() + class Foo: + pass + + @decorate(x=0, y=False) + class Bar: + pass + """) + self.assertTypesMatchPytd(ty, """ + from typing import Callable, TypeVar + Fn = TypeVar('Fn', bound=Callable) + def decorate(**options) -> Callable[[Fn], Fn]: ... + def f(x: int) -> str: ... + def g() -> float: ... + class Foo: ... + class Bar: ... + """) + with file_utils.Tempdir() as d: + d.create_file("foo.pyi", pytd_utils.Print(ty)) + ty = self.Infer(""" + import foo + + @foo.decorate() + def f() -> None: + return None + + @foo.decorate(z=42) + def g(x: int, y: int) -> int: + return x + y + + @foo.decorate() + class Foo: + pass + + @foo.decorate(z=42) + class Bar: + pass + """, pythonpath=[d.path]) + self.assertTypesMatchPytd(ty, """ + foo: module + def f() -> None: ... + def g(x: int, y: int) -> int: ... + class Foo: ... + class Bar: ... + """) + + def test_identity_or_factory(self): + ty = self.Infer(""" + from typing import Any, Callable, overload, TypeVar + Fn = TypeVar('Fn', bound=Callable[..., Any]) + + @overload + def decorate(fn: Fn) -> Fn: ... + + @overload + def decorate(fn: None = None, **options: Any) -> Callable[[Fn], Fn]: ... + + def decorate(fn=None, **options): + if fn: + return fn + def inner(fn): + return fn + return inner + + @decorate + def f() -> bool: + return True + + @decorate() + def g(x: complex) -> float: + return x.real + + @decorate + class Foo: + pass + + @decorate(x=3.14) + class Bar: + pass + """) + self.assertTypesMatchPytd(ty, """ + from typing import Callable, overload, TypeVar + Fn = TypeVar('Fn', bound=Callable) + + @overload + def decorate(fn: Fn) -> Fn: ... + @overload + def decorate(fn: None = ..., **options) -> Callable[[Fn], Fn]: ... + + def f() -> bool: ... + def g(x: complex) -> float: ... + class Foo: ... + class Bar: ... + """) + with file_utils.Tempdir() as d: + d.create_file("foo.pyi", pytd_utils.Print(ty)) + ty = self.Infer(""" + import foo + + @foo.decorate + def f(x: float) -> str: + return str(x) + + @foo.decorate(y=False, z=None) + def g(x: int, y: float) -> float: + return x + y + + @foo.decorate + class Foo: + pass + + @foo.decorate() + class Bar: + pass + """, pythonpath=[d.path]) + self.assertTypesMatchPytd(ty, """ + foo: module + def f(x: float) -> str: ... + def g(x: int, y: float) -> float: ... + class Foo: ... + class Bar: ... + """) + + class DecoratorsTest(test_base.TargetPython3BasicTest): """Test decorators.""" @@ -38,29 +229,5 @@ def func(self, x): x.func(12) """) - def test_class_decorators(self): - ty = self.Infer(""" - from typing import Callable, TypeVar - C = TypeVar('C') - def decorator(cls: C) -> C: - return cls - def decorator_factory() -> Callable[[C], C]: - return lambda x: x - @decorator - class Foo: - pass - @decorator_factory() - class Bar: - pass - """) - self.assertTypesMatchPytd(ty, """ - from typing import Callable, TypeVar - C = TypeVar('C') - def decorator(cls: C) -> C: ... - def decorator_factory() -> Callable[[C], C]: ... - class Foo: ... - class Bar: ... - """) - test_base.main(globals(), __name__ == "__main__") diff --git a/pytype/tests/py3/test_functions.py b/pytype/tests/py3/test_functions.py index 18b6af293..5757e794c 100644 --- a/pytype/tests/py3/test_functions.py +++ b/pytype/tests/py3/test_functions.py @@ -1,7 +1,6 @@ """Test functions.""" from pytype import file_utils -from pytype.pytd import pytd_utils from pytype.tests import test_base @@ -327,35 +326,6 @@ def g(*args): """, pythonpath=[d.path]) self.assertErrorRegexes(errors, {"e": r"int.*str"}) - def test_identity_decorator(self): - ty = self.Infer(""" - from typing import TypeVar - T = TypeVar("T") - def decorate(func: T) -> T: - return func - @decorate - def f(): - return 0 - """) - self.assertTypesMatchPytd(ty, """ - from typing import TypeVar - T = TypeVar("T") - def decorate(func: T) -> T: ... - def f() -> int: ... - """) - with file_utils.Tempdir() as d: - d.create_file("foo.pyi", pytd_utils.Print(ty)) - ty2 = self.Infer(""" - import foo - @foo.decorate - def g(): - return "" - """, pythonpath=[d.path]) - self.assertTypesMatchPytd(ty2, """ - foo: module - def g() -> str: ... - """) - def test_function_type(self): self.Check(""" import types