[WIP] Automated 3.9 update

ast3/Custom/ta3_compat.c

@@ -0,0 +1,66 @@
+#include "../Include/ta3_compat.h"
+
+// Backport of _PyObject_LookupAttr for 3.6
+int
+_Pegen_PyObject_LookupAttr(PyObject *v, PyObject *name, PyObject **result)
+{
+    PyTypeObject *tp = Py_TYPE(v);
+
+    if (!PyUnicode_Check(name)) {
+        PyErr_Format(PyExc_TypeError,
+                     "attribute name must be string, not '%.200s'",
+                     Py_TYPE(name)->tp_name);
+        *result = NULL;
+        return -1;
+    }
+
+    if (tp->tp_getattro == PyObject_GenericGetAttr) {
+        *result = PyObject_GenericGetAttr(v, name);
+        if (*result != NULL) {
+            return 1;
+        }
+        if (PyErr_Occurred()) {
+            return -1;
+        }
+        return 0;
+    }
+    if (tp->tp_getattro != NULL) {
+        *result = (*tp->tp_getattro)(v, name);
+    }
+    else if (tp->tp_getattr != NULL) {
+        const char *name_str = PyUnicode_AsUTF8(name);
+        if (name_str == NULL) {
+            *result = NULL;
+            return -1;
+        }
+        *result = (*tp->tp_getattr)(v, (char *)name_str);
+    }
+    else {
+        *result = NULL;
+        return 0;
+    }
+
+    if (*result != NULL) {
+        return 1;
+    }
+    if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
+        return -1;
+    }
+    PyErr_Clear();
+    return 0;
+}
+
+// Backports of _PyType_Name for Python 3.6
+const char *
+_Ta3Type_Name(PyTypeObject *type)
+{
+    assert(type->tp_name != NULL);
+    const char *s = strrchr(type->tp_name, '.');
+    if (s == NULL) {
+        s = type->tp_name;
+    }
+    else {
+        s++;
+    }
+    return s;
+}

ast3/Custom/typed_ast.c

@@ -1,18 +1,15 @@
+#define Py_PYTHON_AST_H
 #include "Python.h"
 #include "../Include/Python-ast.h"
-#include "../Include/node.h"
-#include "../Include/grammar.h"
 #include "../Include/token.h"
-#include "../Include/ast.h"
-#include "../Include/parsetok.h"
-#include "../Include/errcode.h"
-#include "../Include/graminit.h"
+#include "errcode.h"
+#include "../Include/ta3_compat.h"
+#include "../Include/pegen_interface.h"
+#include "../Parser/pegen/pegen.h"
 
-extern grammar _Ta3Parser_Grammar; /* from graminit.c */
-
-// from Python/bltinmodule.c
-static const char *
-source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
+// from Python/pythonrun.c
+const char *
+_Pegen_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PegenCompilerFlags *cf, PyObject **cmd_copy)
 {
     const char *str;
     Py_ssize_t size;
@@ -46,241 +43,32 @@ source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompil
     }
     else {
         PyErr_Format(PyExc_TypeError,
-          "%s() arg 1 must be a %s object",
-          funcname, what);
+            "%s() arg 1 must be a %s object",
+            funcname, what);
         return NULL;
     }
 
     if (strlen(str) != (size_t)size) {
         PyErr_SetString(PyExc_ValueError,
-                        "source code string cannot contain null bytes");
+            "source code string cannot contain null bytes");
         Py_CLEAR(*cmd_copy);
         return NULL;
     }
     return str;
 }
 
-// from Python/pythonrun.c
-/* compute parser flags based on compiler flags */
-static int PARSER_FLAGS(PyCompilerFlags *flags)
-{
-    int parser_flags = 0;
-    if (!flags)
-        return 0;
-    if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT)
-        parser_flags |= PyPARSE_DONT_IMPLY_DEDENT;
-    if (flags->cf_flags & PyCF_IGNORE_COOKIE)
-        parser_flags |= PyPARSE_IGNORE_COOKIE;
-    if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL)
-        parser_flags |= PyPARSE_BARRY_AS_BDFL;
-    return parser_flags;
-}
-
-// from Python/pythonrun.c
-/* Set the error appropriate to the given input error code (see errcode.h) */
-static void
-err_input(perrdetail *err)
-{
-    PyObject *v, *w, *errtype, *errtext;
-    PyObject *msg_obj = NULL;
-    char *msg = NULL;
-    int offset = err->offset;
-
-    errtype = PyExc_SyntaxError;
-    switch (err->error) {
-    case E_ERROR:
-        return;
-    case E_SYNTAX:
-        errtype = PyExc_IndentationError;
-        if (err->expected == INDENT)
-            msg = "expected an indented block";
-        else if (err->token == INDENT)
-            msg = "unexpected indent";
-        else if (err->token == DEDENT)
-            msg = "unexpected unindent";
-        else {
-            errtype = PyExc_SyntaxError;
-            if (err->token == TYPE_COMMENT)
-              msg = "misplaced type annotation";
-            else
-            msg = "invalid syntax";
-        }
-        break;
-    case E_TOKEN:
-        msg = "invalid token";
-        break;
-    case E_EOFS:
-        msg = "EOF while scanning triple-quoted string literal";
-        break;
-    case E_EOLS:
-        msg = "EOL while scanning string literal";
-        break;
-    case E_INTR:
-        if (!PyErr_Occurred())
-            PyErr_SetNone(PyExc_KeyboardInterrupt);
-        goto cleanup;
-    case E_NOMEM:
-        PyErr_NoMemory();
-        goto cleanup;
-    case E_EOF:
-        msg = "unexpected EOF while parsing";
-        break;
-    case E_TABSPACE:
-        errtype = PyExc_TabError;
-        msg = "inconsistent use of tabs and spaces in indentation";
-        break;
-    case E_OVERFLOW:
-        msg = "expression too long";
-        break;
-    case E_DEDENT:
-        errtype = PyExc_IndentationError;
-        msg = "unindent does not match any outer indentation level";
-        break;
-    case E_TOODEEP:
-        errtype = PyExc_IndentationError;
-        msg = "too many levels of indentation";
-        break;
-    case E_DECODE: {
-        PyObject *type, *value, *tb;
-        PyErr_Fetch(&type, &value, &tb);
-        msg = "unknown decode error";
-        if (value != NULL)
-            msg_obj = PyObject_Str(value);
-        Py_XDECREF(type);
-        Py_XDECREF(value);
-        Py_XDECREF(tb);
-        break;
-    }
-    case E_LINECONT:
-        msg = "unexpected character after line continuation character";
-        break;
-
-    case E_IDENTIFIER:
-        msg = "invalid character in identifier";
-        break;
-    case E_BADSINGLE:
-        msg = "multiple statements found while compiling a single statement";
-        break;
-    default:
-        fprintf(stderr, "error=%d\n", err->error);
-        msg = "unknown parsing error";
-        break;
-    }
-    /* err->text may not be UTF-8 in case of decoding errors.
-       Explicitly convert to an object. */
-    if (!err->text) {
-        errtext = Py_None;
-        Py_INCREF(Py_None);
-    } else {
-        errtext = PyUnicode_DecodeUTF8(err->text, err->offset,
-                                       "replace");
-        if (errtext != NULL) {
-            Py_ssize_t len = strlen(err->text);
-            offset = (int)PyUnicode_GET_LENGTH(errtext);
-            if (len != err->offset) {
-                Py_DECREF(errtext);
-                errtext = PyUnicode_DecodeUTF8(err->text, len,
-                                               "replace");
-            }
-        }
-    }
-    v = Py_BuildValue("(OiiN)", err->filename,
-                      err->lineno, offset, errtext);
-    if (v != NULL) {
-        if (msg_obj)
-            w = Py_BuildValue("(OO)", msg_obj, v);
-        else
-            w = Py_BuildValue("(sO)", msg, v);
-    } else
-        w = NULL;
-    Py_XDECREF(v);
-    PyErr_SetObject(errtype, w);
-    Py_XDECREF(w);
-cleanup:
-    Py_XDECREF(msg_obj);
-    if (err->text != NULL) {
-        PyObject_FREE(err->text);
-        err->text = NULL;
-    }
-}
-
-// from Python/pythonrun.c
-static void
-err_free(perrdetail *err)
-{
-    Py_CLEAR(err->filename);
-}
-
-// from Python/pythonrun.c
-node *
-Ta3Parser_SimpleParseStringFlagsFilename(const char *str, const char *filename,
-                                         int start, int flags)
-{
-    perrdetail err;
-    node *n = Ta3Parser_ParseStringFlagsFilename(str, filename,
-                            &_Ta3Parser_Grammar, start, &err, flags);
-    if (n == NULL)
-        err_input(&err);
-    err_free(&err);
-    return n;
-}
-
-/* update compiler and parser flags based on feature version */
-void
-_Ta3Parser_UpdateFlags(PyCompilerFlags *flags, int *iflags, int feature_version)
-{
-    *iflags = PARSER_FLAGS(flags);
-    if (feature_version >= 7)
-        *iflags |= PyPARSE_ASYNC_ALWAYS;
-    flags->cf_flags |= *iflags & PyCF_MASK;
-}
-
-// copy of PyParser_ASTFromStringObject in Python/pythonrun.c
-/* Preferred access to parser is through AST. */
-static mod_ty
-string_object_to_c_ast(const char *s, PyObject *filename, int start,
-                             PyCompilerFlags *flags, int feature_version,
-                             PyArena *arena)
-{
-    mod_ty mod;
-    PyCompilerFlags localflags;
-    perrdetail err;
-    node *n;
-    int iflags;
-
-    if (flags == NULL) {
-        localflags.cf_flags = 0;
-        flags = &localflags;
-    }
-    _Ta3Parser_UpdateFlags(flags, &iflags, feature_version);
-    n = Ta3Parser_ParseStringObject(s, filename,
-                                    &_Ta3Parser_Grammar, start, &err,
-                                    &iflags);
-    if (n) {
-        flags->cf_flags |= iflags & PyCF_MASK;
-        mod = Ta3AST_FromNodeObject(n, flags, filename, feature_version, arena);
-        Ta3Node_Free(n);
-    }
-    else {
-        err_input(&err);
-        mod = NULL;
-    }
-    err_free(&err);
-    return mod;
-}
-
 // adapted from Py_CompileStringObject in Python/pythonrun.c
 static PyObject *
-string_object_to_py_ast(const char *str, PyObject *filename, int start,
-                       PyCompilerFlags *flags, int feature_version)
+string_object_to_py_ast(const char *str, PyObject *filename, int mode,
+                       PegenCompilerFlags *flags)
 {
     mod_ty mod;
     PyObject *result;
     PyArena *arena = PyArena_New();
     if (arena == NULL)
         return NULL;
 
-    mod = string_object_to_c_ast(str, filename, start, flags, feature_version, arena);
+    mod = Ta3Pegen_ASTFromStringObject(str, filename, mode, flags, arena);
     if (mod == NULL) {
         PyArena_Free(arena);
         return NULL;
@@ -301,11 +89,12 @@ ast3_parse_impl(PyObject *source,
     PyObject *source_copy;
     const char *str;
     int compile_mode = -1;
-    PyCompilerFlags cf;
-    int start[] = {file_input, eval_input, single_input, func_type_input};
+    PegenCompilerFlags cf;
+    //int start[] = {file_input, eval_input, single_input, func_type_input};
     PyObject *result;
 
     cf.cf_flags = PyCF_ONLY_AST | PyCF_SOURCE_IS_UTF8;
+    cf.cf_feature_version = feature_version;
 
     if (strcmp(mode, "exec") == 0)
         compile_mode = 0;
@@ -321,11 +110,11 @@ ast3_parse_impl(PyObject *source,
         goto error;
     }
 
-    str = source_as_string(source, "parse", "string or bytes", &cf, &source_copy);
+    str = _Pegen_SourceAsString(source, "parse", "string or bytes", &cf, &source_copy);
     if (str == NULL)
         goto error;
 
-    result = string_object_to_py_ast(str, filename, start[compile_mode], &cf, feature_version);
+    result = string_object_to_py_ast(str, filename, compile_mode, &cf);
     Py_XDECREF(source_copy);
     goto finally;
 
@@ -336,7 +125,7 @@ ast3_parse_impl(PyObject *source,
     return result;
 }
 
-// adapted from builtin_compile in Python/clinic/bltinmodule.c.h
+// adapted from old typed_ast based on builtin_compile in Python/clinic/bltinmodule.c.h
 PyObject *
 ast3_parse(PyObject *self, PyObject *args)
 {