gh-103763: Implement PEP 695 (#103764)

This implements PEP 695, Type Parameter Syntax. It adds support for:

- Generic functions (def func[T](): ...)
- Generic classes (class X[T](): ...)
- Type aliases (type X = ...)
- New scoping when the new syntax is used within a class body
- Compiler and interpreter changes to support the new syntax and scoping rules 

Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com>
Co-authored-by: Eric Traut <eric@traut.com>
Co-authored-by: Larry Hastings <larry@hastings.org>
Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>

Author: 5 people

Doc/library/ast.rst

@@ -1724,6 +1724,7 @@ Function and class definitions
             body=[
                 FunctionDef(
                     name='f',
+                    typeparams=[],
                     args=arguments(
                         posonlyargs=[],
                         args=[
@@ -1847,6 +1848,7 @@ Function and class definitions
             body=[
                 ClassDef(
                     name='Foo',
+                    typeparams=[],
                     bases=[
                         Name(id='base1', ctx=Load()),
                         Name(id='base2', ctx=Load())],
@@ -1885,6 +1887,7 @@ Async and await
         body=[
             AsyncFunctionDef(
                 name='f',
+                typeparams=[],
                 args=arguments(
                     posonlyargs=[],
                     args=[],

Grammar/python.gram

@@ -112,6 +112,7 @@ simple_stmts[asdl_stmt_seq*]:
 # will throw a SyntaxError.
 simple_stmt[stmt_ty] (memo):
     | assignment
+    | &"type" type_alias
     | e=star_expressions { _PyAST_Expr(e, EXTRA) }
     | &'return' return_stmt
     | &('import' | 'from') import_stmt
@@ -252,8 +253,8 @@ class_def[stmt_ty]:
 
 class_def_raw[stmt_ty]:
     | invalid_class_def_raw
-    | 'class' a=NAME b=['(' z=[arguments] ')' { z }] ':' c=block {
-        _PyAST_ClassDef(a->v.Name.id,
+    | 'class' a=NAME t=[type_params] b=['(' z=[arguments] ')' { z }] ':' c=block {
+        _PyAST_ClassDef(a->v.Name.id, t,
                      (b) ? ((expr_ty) b)->v.Call.args : NULL,
                      (b) ? ((expr_ty) b)->v.Call.keywords : NULL,
                      c, NULL, EXTRA) }
@@ -267,16 +268,16 @@ function_def[stmt_ty]:
 
 function_def_raw[stmt_ty]:
     | invalid_def_raw
-    | 'def' n=NAME &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
-        _PyAST_FunctionDef(n->v.Name.id,
+    | 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
+        _PyAST_FunctionDef(n->v.Name.id, t,
                         (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)),
                         b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) }
-    | ASYNC 'def' n=NAME &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
+    | ASYNC 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
         CHECK_VERSION(
             stmt_ty,
             5,
             "Async functions are",
-            _PyAST_AsyncFunctionDef(n->v.Name.id,
+            _PyAST_AsyncFunctionDef(n->v.Name.id, t,
                             (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)),
                             b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA)
         ) }
@@ -628,6 +629,39 @@ keyword_patterns[asdl_seq*]:
 keyword_pattern[KeyPatternPair*]:
     | arg=NAME '=' value=pattern { _PyPegen_key_pattern_pair(p, arg, value) }
 
+# Type statement
+# ---------------
+
+type_alias[stmt_ty]:
+    | "type" n=NAME t=[type_params] '=' b=expression {
+        CHECK_VERSION(stmt_ty, 12, "Type statement is",
+        _PyAST_TypeAlias(CHECK(expr_ty, _PyPegen_set_expr_context(p, n, Store)), t, b, EXTRA)) }
+
+# Type parameter declaration
+# --------------------------
+
+type_params[asdl_typeparam_seq*]: '[' t=type_param_seq  ']' {
+        CHECK_VERSION(asdl_typeparam_seq *, 12, "Type parameter lists are", t) }
+
+type_param_seq[asdl_typeparam_seq*]: a[asdl_typeparam_seq*]=','.type_param+ [','] { a }
+
+type_param[typeparam_ty] (memo):
+    | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) }
+    | '*' a=NAME colon=":" e=expression {
+            RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
+                ? "cannot use constraints with TypeVarTuple"
+                : "cannot use bound with TypeVarTuple")
+        }
+    | '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) }
+    | '**' a=NAME colon=":" e=expression {
+            RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
+                ? "cannot use constraints with ParamSpec"
+                : "cannot use bound with ParamSpec")
+        }
+    | '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) }
+
+type_param_bound[expr_ty]: ":" e=expression { e }
+
 # EXPRESSIONS
 # -----------
 

Include/cpython/funcobject.h

@@ -41,6 +41,7 @@ typedef struct {
     PyObject *func_weakreflist; /* List of weak references */
     PyObject *func_module;      /* The __module__ attribute, can be anything */
     PyObject *func_annotations; /* Annotations, a dict or NULL */
+    PyObject *func_typeparams;  /* Tuple of active type variables or NULL */
     vectorcallfunc vectorcall;
     /* Version number for use by specializer.
      * Can set to non-zero when we want to specialize.

Include/internal/pycore_ast.h

@@ -17,6 +17,8 @@ struct _py_func_state {
 extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr);
 
 extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func);
+extern PyObject *_Py_set_function_type_params(
+    PyThreadState* unused, PyObject *func, PyObject *type_params);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_ast_state.h

@@ -68,6 +68,14 @@ struct _Py_interp_cached_objects {
     PyObject *type_slots_pname;
     pytype_slotdef *type_slots_ptrs[MAX_EQUIV];
 
+    /* TypeVar and related types */
+    PyTypeObject *generic_type;
+    PyTypeObject *typevar_type;
+    PyTypeObject *typevartuple_type;
+    PyTypeObject *paramspec_type;
+    PyTypeObject *paramspecargs_type;
+    PyTypeObject *paramspeckwargs_type;
+    PyTypeObject *typealias_type;
 };
 
 #define _Py_INTERP_STATIC_OBJECT(interp, NAME) \