Skip to content

Commit

Permalink
Merge pull request #579 from google/google_sync
Browse files Browse the repository at this point in the history
Google sync
  • Loading branch information
rchen152 authored May 5, 2020
2 parents 9c6e5f1 + 4ec8cbb commit c3cf02e
Show file tree
Hide file tree
Showing 24 changed files with 447 additions and 1,045 deletions.
19 changes: 10 additions & 9 deletions pytype/abstract.py
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,9 @@ def get_class(self):
# See Py_TYPE() in Include/object.h
if self.cls:
return self.cls
elif isinstance(self, InterpreterClass):
return ParameterizedClass(
self.vm.convert.type_type, {abstract_utils.T: self}, self.vm)
elif isinstance(self, (AnnotationClass, mixin.Class)):
return self.vm.convert.type_type

Expand Down Expand Up @@ -3119,7 +3122,7 @@ def call(self, node, func, args, new_locals=False, alias_map=None):
for name in callargs:
if (name in annotations and (not self.is_attribute_of_class or
self.argcount(node) == 0 or
name != self.signature.param_names[0])):
name != sig.param_names[0])):
extra_key = (self.get_first_opcode(), name)
node, callargs[name] = self.vm.init_class(
node, annotations[name], extra_key=extra_key)
Expand All @@ -3138,8 +3141,7 @@ def call(self, node, func, args, new_locals=False, alias_map=None):
# as incomplete.
self._set_callself_maybe_missing_members()
return node, self.vm.new_unsolvable(node)
self_var = (self.signature.param_names and
callargs.get(self.signature.param_names[0]))
self_var = sig.param_names and callargs.get(sig.param_names[0])
caller_is_abstract = abstract_utils.check_classes(
self_var, lambda cls: cls.is_abstract)
caller_is_protocol = abstract_utils.check_classes(
Expand Down Expand Up @@ -3181,7 +3183,7 @@ def call(self, node, func, args, new_locals=False, alias_map=None):
ret = old_ret.AssignToNewVariable(node)
if self._store_call_records:
# Even if the call is cached, we might not have been recording it.
self._call_records.append((sig, callargs, ret, node))
self._call_records.append((callargs, ret, node))
return node, ret
if self.code.has_generator():
generator = Generator(frame, self.vm)
Expand Down Expand Up @@ -3212,15 +3214,15 @@ def call(self, node, func, args, new_locals=False, alias_map=None):
node_after_call = abstract_utils.apply_mutations(node_after_call, mutations)
self._call_cache[callkey] = ret, self.vm.remaining_depth()
if self._store_call_records or self.vm.store_all_calls:
self._call_records.append((sig, callargs, ret, node_after_call))
self._call_records.append((callargs, ret, node_after_call))
self.last_frame = frame
return node_after_call, ret

def get_call_combinations(self, node):
"""Get this function's call records."""
all_combinations = []
signature_data = set()
for sig, callargs, ret, node_after_call in self._call_records:
for callargs, ret, node_after_call in self._call_records:
try:
combinations = cfg_utils.variable_product_dict(callargs)
except cfg_utils.TooComplexError:
Expand All @@ -3241,15 +3243,14 @@ def get_call_combinations(self, node):
node_after_call.HasCombination(values)):
signature_data.add(data)
all_combinations.append(
(sig, node_after_call, combination, return_value))
(node_after_call, combination, return_value))
if not all_combinations:
# Fallback: Generate signatures only from the definition of the
# method, not the way it's being used.
param_binding = self.vm.convert.unsolvable.to_binding(node)
params = collections.defaultdict(lambda: param_binding)
ret = self.vm.convert.unsolvable.to_binding(node)
for f in self.signature_functions():
all_combinations.append((f.signature, node, params, ret))
all_combinations.append((node, params, ret))
return all_combinations

def get_positional_names(self):
Expand Down
8 changes: 4 additions & 4 deletions pytype/config_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,12 +142,12 @@ def test_dependency(self):

def test_subset(self):
input_options = datatypes.SimpleNamespace(
pythonpath=".", python_version="3.4")
pythonpath=".", python_version="3.5")
config.Postprocessor(
{"python_version"}, input_options, self.output_options).process()
with self.assertRaises(AttributeError):
_ = self.output_options.pythonpath # not processed
self.assertTupleEqual(self.output_options.python_version, (3, 4))
self.assertTupleEqual(self.output_options.python_version, (3, 5))

def test_error(self):
input_options = datatypes.SimpleNamespace(check=True, output="test.pyi")
Expand All @@ -157,12 +157,12 @@ def test_error(self):

def test_inplace(self):
input_options = datatypes.SimpleNamespace(
disable="import-error,attribute-error", python_version="3.4")
disable="import-error,attribute-error", python_version="3.5")
config.Postprocessor(
{"disable", "python_version"}, input_options).process()
self.assertSequenceEqual(
input_options.disable, ["import-error", "attribute-error"])
self.assertTupleEqual(input_options.python_version, (3, 4))
self.assertTupleEqual(input_options.python_version, (3, 5))

def test_typeshed_default(self):
input_options = datatypes.SimpleNamespace(
Expand Down
2 changes: 1 addition & 1 deletion pytype/directors.py
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ def _parse_source(self, src):
open_decorator = True
elif tok == tokenize.NAME:
if open_decorator and token.string in ("class", "def"):
self.decorators.add(lineno - 1)
self.decorators.add(lineno)
open_decorator = False
if token.string == "def":
last_function_definition = _FunctionDefinition.start(lineno)
Expand Down
6 changes: 3 additions & 3 deletions pytype/directors_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,8 +405,8 @@ def bar():
pass
""")
self.assertEqual({
6, # real_decorator
13 # decorator
7, # real_decorator
14 # decorator
}, self._director._decorators)

def test_stacked_decorators(self):
Expand All @@ -421,7 +421,7 @@ class A:
pass
""")
self.assertEqual({
7 # foo
8 # foo
}, self._director._decorators)


Expand Down
101 changes: 55 additions & 46 deletions pytype/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -365,14 +365,13 @@ def uninitialized_annotations_to_instance_types(self, node, annots, members):
yield name, pytd_utils.JoinTypes(
value.get_instance_type(node) for value in annots[name].data)

def _function_call_to_return_type(
self, sig, node, v, seen_return, num_returns):
def _function_call_to_return_type(self, node, v, seen_return, num_returns):
"""Get a function call's pytd return type."""
if sig.has_return_annotation:
if v.signature.has_return_annotation:
if v.is_coroutine():
ret = abstract.Coroutine.make(self.vm, v, node).to_type(node)
else:
ret = sig.annotations["return"].get_instance_type(node)
ret = v.signature.annotations["return"].get_instance_type(node)
else:
ret = seen_return.data.to_type(node)
if isinstance(ret, pytd.NothingType) and num_returns == 1:
Expand All @@ -382,49 +381,59 @@ def _function_call_to_return_type(
assert isinstance(seen_return.data, typing_overlay.NoReturn)
return ret

def _function_to_def(self, node, v, function_name):
"""Convert an InterpreterFunction to a PyTD definition."""
signatures = []
combinations = v.get_call_combinations(node)
for sig, node_after, combination, return_value in combinations:
params = []
for i, (name, kwonly, optional) in enumerate(v.get_parameters()):
if i < v.nonstararg_count and name in sig.annotations:
t = sig.annotations[name].get_instance_type(node_after)
else:
t = combination[name].data.to_type(node_after)
# Python uses ".0" etc. for the names of parameters that are tuples,
# like e.g. in: "def f((x, y), z)".
params.append(
pytd.Parameter(name.replace(".", "_"), t, kwonly, optional, None))
ret = self._function_call_to_return_type(
sig, node_after, v, return_value, len(combinations))
if v.has_varargs():
if sig.varargs_name in sig.annotations:
annot = sig.annotations[sig.varargs_name]
typ = annot.get_instance_type(node_after)
else:
typ = pytd.NamedType("__builtin__.tuple")
starargs = pytd.Parameter(sig.varargs_name, typ, False, True, None)
def _function_call_combination_to_signature(
self, func, call_combination, num_combinations):
node_after, combination, return_value = call_combination
params = []
for i, (name, kwonly, optional) in enumerate(func.get_parameters()):
if i < func.nonstararg_count and name in func.signature.annotations:
t = func.signature.annotations[name].get_instance_type(node_after)
else:
starargs = None
if v.has_kwargs():
if sig.kwargs_name in sig.annotations:
annot = sig.annotations[sig.kwargs_name]
typ = annot.get_instance_type(node_after)
else:
typ = pytd.NamedType("__builtin__.dict")
starstarargs = pytd.Parameter(sig.kwargs_name, typ, False, True, None)
t = combination[name].data.to_type(node_after)
# Python uses ".0" etc. for the names of parameters that are tuples,
# like e.g. in: "def f((x, y), z)".
params.append(
pytd.Parameter(name.replace(".", "_"), t, kwonly, optional, None))
ret = self._function_call_to_return_type(
node_after, func, return_value, num_combinations)
if func.has_varargs():
if func.signature.varargs_name in func.signature.annotations:
annot = func.signature.annotations[func.signature.varargs_name]
typ = annot.get_instance_type(node_after)
else:
starstarargs = None
signatures.append(pytd.Signature(
params=tuple(params),
starargs=starargs,
starstarargs=starstarargs,
return_type=ret,
exceptions=(), # TODO(kramm): record exceptions
template=()))
typ = pytd.NamedType("__builtin__.tuple")
starargs = pytd.Parameter(
func.signature.varargs_name, typ, False, True, None)
else:
starargs = None
if func.has_kwargs():
if func.signature.kwargs_name in func.signature.annotations:
annot = func.signature.annotations[func.signature.kwargs_name]
typ = annot.get_instance_type(node_after)
else:
typ = pytd.NamedType("__builtin__.dict")
starstarargs = pytd.Parameter(
func.signature.kwargs_name, typ, False, True, None)
else:
starstarargs = None
return pytd.Signature(
params=tuple(params),
starargs=starargs,
starstarargs=starstarargs,
return_type=ret,
exceptions=(), # TODO(kramm): record exceptions
template=())

def _function_to_def(self, node, v, function_name):
"""Convert an InterpreterFunction to a PyTD definition."""
signatures = []
for func in v.signature_functions():
combinations = func.get_call_combinations(node)
num_combinations = len(combinations)
signatures.extend(
self._function_call_combination_to_signature(
func, combination, num_combinations)
for combination in combinations)
return pytd.Function(name=function_name,
signatures=tuple(signatures),
kind=pytd.METHOD,
Expand Down Expand Up @@ -474,9 +483,9 @@ def _function_to_return_types(self, node, fvar):
for val in options:
if isinstance(val, abstract.InterpreterFunction):
combinations = val.get_call_combinations(node)
for sig, node_after, _, return_value in combinations:
for node_after, _, return_value in combinations:
types.append(self._function_call_to_return_type(
sig, node_after, val, return_value, len(combinations)))
node_after, val, return_value, len(combinations)))
elif isinstance(val, abstract.PyTDFunction):
types.extend(sig.pytd_sig.return_type for sig in val.signatures)
else:
Expand Down
4 changes: 4 additions & 0 deletions pytype/overlays/classgen.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,10 @@ def get_class_locals(self, cls, allow_methods, ordering):
# TODO(rechen): Once we drop Python 2 support, either use a normal dict or
# replace key deletion with OrderedDict.move_to_end().
out = collections.OrderedDict()
if cls.name not in self.vm.local_ops:
# See TestAttribPy3.test_cannot_decorate in tests/py3/test_attr.py. The
# class will not be in local_ops if a previous decorator hides it.
return out
for op in self.vm.local_ops[cls.name]:
if is_dunder(op.name):
continue
Expand Down
2 changes: 1 addition & 1 deletion pytype/pyc/loadmarshal_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ def test_load_code(self):
b'z\4test' # name: 'test.py'
b'\6\0\0\0' # first line no: 6
b'N') # lnotab: None
code = self.load(co, python_version=(3, 4))
code = self.load(co, python_version=(3, 5))
self.assertEqual(code.co_argcount, 1)
self.assertEqual(code.co_kwonlyargcount, 2)
self.assertEqual(code.co_nlocals, 3)
Expand Down
30 changes: 11 additions & 19 deletions pytype/pyc/opcodes.py
Original file line number Diff line number Diff line change
Expand Up @@ -957,7 +957,7 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args
147: MAP_ADD,
}

python3_mapping = {
python_3_5_mapping = {
1: POP_TOP,
2: ROT_TWO,
3: ROT_THREE,
Expand All @@ -968,6 +968,8 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args
11: UNARY_NEGATIVE,
12: UNARY_NOT,
15: UNARY_INVERT,
16: BINARY_MATRIX_MULTIPLY,
17: INPLACE_MATRIX_MULTIPLY,
19: BINARY_POWER,
20: BINARY_MULTIPLY,
22: BINARY_MODULO,
Expand All @@ -981,7 +983,6 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args
50: GET_AITER,
51: GET_ANEXT,
52: BEFORE_ASYNC_WITH,
54: STORE_MAP, # removed in Python 3.5.2
55: INPLACE_ADD,
56: INPLACE_SUBTRACT,
57: INPLACE_MULTIPLY,
Expand All @@ -995,7 +996,7 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args
66: BINARY_OR,
67: INPLACE_POWER,
68: GET_ITER,
69: STORE_LOCALS, # removed in Python 3.4
69: GET_YIELD_FROM_ITER,
70: PRINT_EXPR,
71: LOAD_BUILD_CLASS, # PRINT_ITEM in Python 2
72: YIELD_FROM, # PRINT_NEWLINE in Python 2
Expand All @@ -1006,7 +1007,8 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args
78: INPLACE_XOR,
79: INPLACE_OR,
80: BREAK_LOOP,
81: WITH_CLEANUP,
81: WITH_CLEANUP_START,
82: WITH_CLEANUP_FINISH,
83: RETURN_VALUE,
84: IMPORT_STAR,
86: YIELD_VALUE,
Expand Down Expand Up @@ -1064,6 +1066,11 @@ class CALL_METHOD(OpcodeWithArg): # Arg: #args
146: SET_ADD,
147: MAP_ADD,
148: LOAD_CLASSDEREF, # not in Python 2
149: BUILD_LIST_UNPACK,
150: BUILD_MAP_UNPACK,
151: BUILD_MAP_UNPACK_WITH_CALL,
152: BUILD_TUPLE_UNPACK,
153: BUILD_SET_UNPACK,
154: SETUP_ASYNC_WITH,
}

Expand All @@ -1073,20 +1080,6 @@ def _overlay_mapping(mapping, new_entries):
ret.update(new_entries)
return dict((k, v) for k, v in six.iteritems(ret) if v is not None)

python_3_5_mapping = _overlay_mapping(python3_mapping, {
16: BINARY_MATRIX_MULTIPLY,
17: INPLACE_MATRIX_MULTIPLY,
54: None,
69: GET_YIELD_FROM_ITER, # STORE_LOCALS in Python 3.3
81: WITH_CLEANUP_START, # WITH_CLEANUP in Python 3.4
82: WITH_CLEANUP_FINISH,
149: BUILD_LIST_UNPACK,
150: BUILD_MAP_UNPACK,
151: BUILD_MAP_UNPACK_WITH_CALL,
152: BUILD_TUPLE_UNPACK,
153: BUILD_SET_UNPACK,
})

python_3_6_mapping = _overlay_mapping(python_3_5_mapping, {
85: SETUP_ANNOTATIONS,
127: STORE_ANNOTATION, # removed in Python 3.7
Expand Down Expand Up @@ -1275,7 +1268,6 @@ def dis(data, python_version, *args, **kwargs):
assert major in (2, 3)
mapping = {
(2, 7): python2_mapping,
(3, 4): python3_mapping,
(3, 5): python_3_5_mapping,
(3, 6): python_3_6_mapping,
(3, 7): python_3_7_mapping,
Expand Down
Loading

0 comments on commit c3cf02e

Please sign in to comment.