Skip to content

Commit

Permalink
pythongh-119180: PEP 649 compiler changes (python#119361)
Browse files Browse the repository at this point in the history
  • Loading branch information
JelleZijlstra authored and mrahtz committed Jun 30, 2024
1 parent 22ffaf4 commit 39ab1f0
Show file tree
Hide file tree
Showing 28 changed files with 609 additions and 328 deletions.
2 changes: 1 addition & 1 deletion Include/internal/pycore_global_objects_fini_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Include/internal/pycore_global_strings.h
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ struct _Py_global_strings {
STRUCT_FOR_STR(dot, ".")
STRUCT_FOR_STR(dot_locals, ".<locals>")
STRUCT_FOR_STR(empty, "")
STRUCT_FOR_STR(format, ".format")
STRUCT_FOR_STR(generic_base, ".generic_base")
STRUCT_FOR_STR(json_decoder, "json.decoder")
STRUCT_FOR_STR(kwdefaults, ".kwdefaults")
Expand Down Expand Up @@ -234,7 +235,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(_abstract_)
STRUCT_FOR_ID(_active)
STRUCT_FOR_ID(_align_)
STRUCT_FOR_ID(_annotation)
STRUCT_FOR_ID(_anonymous_)
STRUCT_FOR_ID(_argtypes_)
STRUCT_FOR_ID(_as_parameter_)
Expand Down
1 change: 1 addition & 0 deletions Include/internal/pycore_opcode_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ extern "C" {
#define MAKE_FUNCTION_KWDEFAULTS 0x02
#define MAKE_FUNCTION_ANNOTATIONS 0x04
#define MAKE_FUNCTION_CLOSURE 0x08
#define MAKE_FUNCTION_ANNOTATE 0x10

/* Values used as the oparg for LOAD_COMMON_CONSTANT */
#define CONSTANT_ASSERTIONERROR 0
Expand Down
2 changes: 1 addition & 1 deletion Include/internal/pycore_runtime_init_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 6 additions & 2 deletions Include/internal/pycore_symtable.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@ struct _mod; // Type defined in pycore_ast.h

typedef enum _block_type {
FunctionBlock, ClassBlock, ModuleBlock,
// Used for annotations if 'from __future__ import annotations' is active.
// Annotation blocks cannot bind names and are not evaluated.
// Used for annotations. If 'from __future__ import annotations' is active,
// annotation blocks cannot bind names and are not evaluated. Otherwise, they
// are lazily evaluated (see PEP 649).
AnnotationBlock,
// Used for generics and type aliases. These work mostly like functions
// (see PEP 695 for details). The three different blocks function identically;
Expand Down Expand Up @@ -89,6 +90,7 @@ typedef struct _symtable_entry {
including free refs to globals */
unsigned ste_generator : 1; /* true if namespace is a generator */
unsigned ste_coroutine : 1; /* true if namespace is a coroutine */
unsigned ste_annotations_used : 1; /* true if there are any annotations in this scope */
_Py_comprehension_ty ste_comprehension; /* Kind of comprehension (if any) */
unsigned ste_varargs : 1; /* true if block has varargs */
unsigned ste_varkeywords : 1; /* true if block has varkeywords */
Expand All @@ -110,6 +112,7 @@ typedef struct _symtable_entry {
int ste_end_col_offset; /* end offset of first line of block */
int ste_opt_lineno; /* lineno of last exec or import * */
int ste_opt_col_offset; /* offset of last exec or import * */
struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */
struct symtable *ste_table;
} PySTEntryObject;

Expand All @@ -126,6 +129,7 @@ extern struct symtable* _PySymtable_Build(
PyObject *filename,
_PyFutureFeatures *future);
extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *);
extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **);

extern void _PySymtable_Free(struct symtable *);

Expand Down
3 changes: 0 additions & 3 deletions Include/internal/pycore_unicodeobject_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 1 addition & 7 deletions Lib/inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,13 +220,7 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False):
"""
if isinstance(obj, type):
# class
obj_dict = getattr(obj, '__dict__', None)
if obj_dict and hasattr(obj_dict, 'get'):
ann = obj_dict.get('__annotations__', None)
if isinstance(ann, types.GetSetDescriptorType):
ann = None
else:
ann = None
ann = obj.__annotations__

obj_globals = None
module_name = getattr(obj, '__module__', None)
Expand Down
2 changes: 2 additions & 0 deletions Lib/symtable.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,8 @@ def get_methods(self):
if self.__methods is None:
d = {}
for st in self._table.children:
if st.type == _symtable.TYPE_ANNOTATION:
continue
d[st.name] = 1
self.__methods = tuple(d)
return self.__methods
Expand Down
29 changes: 9 additions & 20 deletions Lib/test/test_dis.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,32 +352,21 @@ def wrap_func_w_kwargs():
dis_annot_stmt_str = """\
0 RESUME 0
2 SETUP_ANNOTATIONS
LOAD_CONST 0 (1)
2 LOAD_CONST 0 (1)
STORE_NAME 0 (x)
LOAD_NAME 1 (int)
LOAD_NAME 2 (__annotations__)
LOAD_CONST 1 ('x')
STORE_SUBSCR
3 LOAD_NAME 3 (fun)
PUSH_NULL
LOAD_CONST 0 (1)
CALL 1
LOAD_NAME 2 (__annotations__)
LOAD_CONST 2 ('y')
STORE_SUBSCR
4 LOAD_CONST 0 (1)
LOAD_NAME 4 (lst)
LOAD_NAME 3 (fun)
LOAD_NAME 1 (lst)
LOAD_NAME 2 (fun)
PUSH_NULL
LOAD_CONST 3 (0)
LOAD_CONST 1 (0)
CALL 1
STORE_SUBSCR
LOAD_NAME 1 (int)
POP_TOP
RETURN_CONST 4 (None)
2 LOAD_CONST 2 (<code object __annotate__ at 0x..., file "<dis>", line 2>)
MAKE_FUNCTION
STORE_NAME 3 (__annotate__)
RETURN_CONST 3 (None)
"""

compound_stmt_str = """\
Expand Down
69 changes: 5 additions & 64 deletions Lib/test/test_grammar.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,16 +306,6 @@ def test_eof_error(self):

var_annot_global: int # a global annotated is necessary for test_var_annot

# custom namespace for testing __annotations__

class CNS:
def __init__(self):
self._dct = {}
def __setitem__(self, item, value):
self._dct[item.lower()] = value
def __getitem__(self, item):
return self._dct[item]


class GrammarTests(unittest.TestCase):

Expand Down Expand Up @@ -446,22 +436,12 @@ class F(C, A):
self.assertEqual(E.__annotations__, {})
self.assertEqual(F.__annotations__, {})


def test_var_annot_metaclass_semantics(self):
class CMeta(type):
@classmethod
def __prepare__(metacls, name, bases, **kwds):
return {'__annotations__': CNS()}
class CC(metaclass=CMeta):
XX: 'ANNOT'
self.assertEqual(CC.__annotations__['xx'], 'ANNOT')

def test_var_annot_module_semantics(self):
self.assertEqual(test.__annotations__, {})
self.assertEqual(ann_module.__annotations__,
{1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float})
{'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float})
self.assertEqual(ann_module.M.__annotations__,
{'123': 123, 'o': type})
{'o': type})
self.assertEqual(ann_module2.__annotations__, {})

def test_var_annot_in_module(self):
Expand All @@ -476,51 +456,12 @@ def test_var_annot_in_module(self):
ann_module3.D_bad_ann(5)

def test_var_annot_simple_exec(self):
gns = {}; lns= {}
gns = {}; lns = {}
exec("'docstring'\n"
"__annotations__[1] = 2\n"
"x: int = 5\n", gns, lns)
self.assertEqual(lns["__annotations__"], {1: 2, 'x': int})
with self.assertRaises(KeyError):
gns['__annotations__']

def test_var_annot_custom_maps(self):
# tests with custom locals() and __annotations__
ns = {'__annotations__': CNS()}
exec('X: int; Z: str = "Z"; (w): complex = 1j', ns)
self.assertEqual(ns['__annotations__']['x'], int)
self.assertEqual(ns['__annotations__']['z'], str)
self.assertEqual(lns["__annotate__"](1), {'x': int})
with self.assertRaises(KeyError):
ns['__annotations__']['w']
nonloc_ns = {}
class CNS2:
def __init__(self):
self._dct = {}
def __setitem__(self, item, value):
nonlocal nonloc_ns
self._dct[item] = value
nonloc_ns[item] = value
def __getitem__(self, item):
return self._dct[item]
exec('x: int = 1', {}, CNS2())
self.assertEqual(nonloc_ns['__annotations__']['x'], int)

def test_var_annot_refleak(self):
# complex case: custom locals plus custom __annotations__
# this was causing refleak
cns = CNS()
nonloc_ns = {'__annotations__': cns}
class CNS2:
def __init__(self):
self._dct = {'__annotations__': cns}
def __setitem__(self, item, value):
nonlocal nonloc_ns
self._dct[item] = value
nonloc_ns[item] = value
def __getitem__(self, item):
return self._dct[item]
exec('X: str', {}, CNS2())
self.assertEqual(nonloc_ns['__annotations__']['x'], str)
gns['__annotate__']

def test_var_annot_rhs(self):
ns = {}
Expand Down
2 changes: 2 additions & 0 deletions Lib/test/test_module/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,8 @@ def test_annotations_are_created_correctly(self):
ann_module4 = import_helper.import_fresh_module(
'test.typinganndata.ann_module4',
)
self.assertFalse("__annotations__" in ann_module4.__dict__)
self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str})
self.assertTrue("__annotations__" in ann_module4.__dict__)
del ann_module4.__annotations__
self.assertFalse("__annotations__" in ann_module4.__dict__)
Expand Down
13 changes: 8 additions & 5 deletions Lib/test/test_opcodes.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,19 @@ class C: pass
def test_use_existing_annotations(self):
ns = {'__annotations__': {1: 2}}
exec('x: int', ns)
self.assertEqual(ns['__annotations__'], {'x': int, 1: 2})
self.assertEqual(ns['__annotations__'], {1: 2})

def test_do_not_recreate_annotations(self):
# Don't rely on the existence of the '__annotations__' global.
with support.swap_item(globals(), '__annotations__', {}):
del globals()['__annotations__']
globals().pop('__annotations__', None)
class C:
del __annotations__
with self.assertRaises(NameError):
x: int
try:
del __annotations__
except NameError:
pass
x: int
self.assertEqual(C.__annotations__, {"x": int})

def test_raise_class_exceptions(self):

Expand Down
5 changes: 4 additions & 1 deletion Lib/test/test_positional_only_arg.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import dis
import pickle
import types
import unittest

from test.support import check_syntax_error
Expand Down Expand Up @@ -440,7 +441,9 @@ def f(x: not (int is int), /): ...
# without constant folding we end up with
# COMPARE_OP(is), IS_OP (0)
# with constant folding we should expect a IS_OP (1)
codes = [(i.opname, i.argval) for i in dis.get_instructions(g)]
code_obj = next(const for const in g.__code__.co_consts
if isinstance(const, types.CodeType) and const.co_name == "__annotate__")
codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)]
self.assertNotIn(('UNARY_NOT', None), codes)
self.assertIn(('IS_OP', 1), codes)

Expand Down
2 changes: 2 additions & 0 deletions Lib/test/test_pyclbr.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ def ismethod(oclass, obj, name):

actualMethods = []
for m in py_item.__dict__.keys():
if m == "__annotate__":
continue
if ismethod(py_item, getattr(py_item, m), m):
actualMethods.append(m)
foundMethods = []
Expand Down
11 changes: 8 additions & 3 deletions Lib/test/test_pydoc/test_pydoc.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,11 @@ class A(builtins.object)
| __weakref__%s
class B(builtins.object)
| Methods defined here:
|
| __annotate__(...)
|
| ----------------------------------------------------------------------
| Data descriptors defined here:
|
| __dict__%s
Expand All @@ -87,8 +92,6 @@ class B(builtins.object)
| Data and other attributes defined here:
|
| NO_MEANING = 'eggs'
|
| __annotations__ = {'NO_MEANING': <class 'str'>}
class C(builtins.object)
| Methods defined here:
Expand Down Expand Up @@ -176,6 +179,9 @@ class A(builtins.object)
list of weak references to the object
class B(builtins.object)
Methods defined here:
__annotate__(...)
----------------------------------------------------------------------
Data descriptors defined here:
__dict__
dictionary for instance variables
Expand All @@ -184,7 +190,6 @@ class B(builtins.object)
----------------------------------------------------------------------
Data and other attributes defined here:
NO_MEANING = 'eggs'
__annotations__ = {'NO_MEANING': <class 'str'>}
class C(builtins.object)
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_pyrepl/test_interact.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self):

def test_no_active_future(self):
console = InteractiveColoredConsole()
source = "x: int = 1; print(__annotations__)"
source = "x: int = 1; print(__annotate__(1))"
f = io.StringIO()
with contextlib.redirect_stdout(f):
result = console.runsource(source)
Expand Down
6 changes: 4 additions & 2 deletions Lib/test/test_symtable.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,12 +205,14 @@ def test_assigned(self):

def test_annotated(self):
st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec')
st2 = st1.get_children()[0]
st2 = st1.get_children()[1]
self.assertEqual(st2.get_type(), "function")
self.assertTrue(st2.lookup('x').is_local())
self.assertTrue(st2.lookup('x').is_annotated())
self.assertFalse(st2.lookup('x').is_global())
st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec')
st4 = st3.get_children()[0]
st4 = st3.get_children()[1]
self.assertEqual(st4.get_type(), "function")
self.assertTrue(st4.lookup('x').is_local())
self.assertFalse(st4.lookup('x').is_annotated())

Expand Down
Loading

0 comments on commit 39ab1f0

Please sign in to comment.