From d188432563b4248c300dd5a97d633d402f9824f2 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Tue, 21 May 2019 20:46:02 -0700 Subject: [PATCH 1/5] Makes symtable.symtable have parity for accepted datatypes for source code as compile() --- Include/pythonrun.h | 13 +++++++ Lib/test/test_symtable.py | 9 +++++ Modules/clinic/symtablemodule.c.h | 22 +++-------- Modules/symtablemodule.c | 25 +++++++++--- Python/bltinmodule.c | 55 ++------------------------ Python/pythonrun.c | 64 +++++++++++++++++++++++++++++-- 6 files changed, 109 insertions(+), 79 deletions(-) diff --git a/Include/pythonrun.h b/Include/pythonrun.h index e83846add981cd..196355cb8f40bf 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -119,10 +119,23 @@ PyAPI_FUNC(struct symtable *) Py_SymtableString( const char *filename, /* decoded from the filesystem encoding */ int start); #ifndef Py_LIMITED_API +PyAPI_FUNC(const char *) _Py_SourceAsString( + PyObject *cmd, + const char *funcname, + const char *what, + PyCompilerFlags *cf, + PyObject **cmd_copy); + PyAPI_FUNC(struct symtable *) Py_SymtableStringObject( const char *str, PyObject *filename, int start); + +PyAPI_FUNC(struct symtable *) _Py_SymtableStringObjectFlags( + const char *str, + PyObject *filename, + int start, + PyCompilerFlags *flags); #endif PyAPI_FUNC(void) PyErr_Print(void); diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 0a1cb8d5b43283..bea2ce120ca96b 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -215,6 +215,15 @@ def test_single(self): def test_exec(self): symbols = symtable.symtable("def f(x): return x", "?", "exec") + def test_bytes(self): + top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec") + self.assertIsNotNone(find_block(top, "Mine")) + + code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n' + + top = symtable.symtable(code, "?", "exec") + self.assertIsNotNone(find_block(top, "\u017d")) + if __name__ == '__main__': unittest.main() diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index 73e340bd462a63..d6021e444fe8f1 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -12,33 +12,21 @@ PyDoc_STRVAR(_symtable_symtable__doc__, {"symtable", (PyCFunction)(void(*)(void))_symtable_symtable, METH_FASTCALL, _symtable_symtable__doc__}, static PyObject * -_symtable_symtable_impl(PyObject *module, const char *str, - PyObject *filename, const char *startstr); +_symtable_symtable_impl(PyObject *module, PyObject *str, PyObject *filename, + const char *startstr); static PyObject * _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - const char *str; + PyObject *str; PyObject *filename; const char *startstr; if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { goto exit; } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("symtable", 1, "str", args[0]); - goto exit; - } - Py_ssize_t str_length; - str = PyUnicode_AsUTF8AndSize(args[0], &str_length); - if (str == NULL) { - goto exit; - } - if (strlen(str) != (size_t)str_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + str = args[0]; if (!PyUnicode_FSDecoder(args[1], &filename)) { goto exit; } @@ -60,4 +48,4 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=be1cca59de019984 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a72eb8187a6bc8aa input=a9049054013a1b77]*/ diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index e8d2f5b582be20..f93cfce902e107 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -14,7 +14,7 @@ module _symtable /*[clinic input] _symtable.symtable - str: str + str: object filename: object(converter='PyUnicode_FSDecoder') startstr: str / @@ -23,13 +23,23 @@ Return symbol and scope dictionaries used internally by compiler. [clinic start generated code]*/ static PyObject * -_symtable_symtable_impl(PyObject *module, const char *str, - PyObject *filename, const char *startstr) -/*[clinic end generated code: output=914b369c9b785956 input=6c615e84d5f408e3]*/ +_symtable_symtable_impl(PyObject *module, PyObject *str, PyObject *filename, + const char *startstr) +/*[clinic end generated code: output=da9d42d2103ea619 input=a8b56a8eee46c26b]*/ { struct symtable *st; PyObject *t; int start; + PyCompilerFlags cf; + PyObject *source_copy = NULL; + + cf.cf_flags = PyCF_SOURCE_IS_UTF8; + cf.cf_feature_version = PY_MINOR_VERSION; + + const char *source = _Py_SourceAsString(str, "symtable", "string or bytes", &cf, &source_copy); + if (source == NULL) { + return NULL; + } if (strcmp(startstr, "exec") == 0) start = Py_file_input; @@ -41,12 +51,15 @@ _symtable_symtable_impl(PyObject *module, const char *str, PyErr_SetString(PyExc_ValueError, "symtable() arg 3 must be 'exec' or 'eval' or 'single'"); Py_DECREF(filename); + Py_XDECREF(source_copy); return NULL; } - st = Py_SymtableStringObject(str, filename, start); + st = _Py_SymtableStringObjectFlags(source, filename, start, &cf); Py_DECREF(filename); - if (st == NULL) + if (st == NULL) { + Py_XDECREF(source_copy); return NULL; + } t = (PyObject *)st->st_top; Py_INCREF(t); PyMem_Free((void *)st->st_future); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 047cca057b41ae..a94d38952234d0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -687,55 +687,6 @@ builtin_chr_impl(PyObject *module, int i) } -static const char * -source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy) -{ - const char *str; - Py_ssize_t size; - Py_buffer view; - - *cmd_copy = NULL; - if (PyUnicode_Check(cmd)) { - cf->cf_flags |= PyCF_IGNORE_COOKIE; - str = PyUnicode_AsUTF8AndSize(cmd, &size); - if (str == NULL) - return NULL; - } - else if (PyBytes_Check(cmd)) { - str = PyBytes_AS_STRING(cmd); - size = PyBytes_GET_SIZE(cmd); - } - else if (PyByteArray_Check(cmd)) { - str = PyByteArray_AS_STRING(cmd); - size = PyByteArray_GET_SIZE(cmd); - } - else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) { - /* Copy to NUL-terminated buffer. */ - *cmd_copy = PyBytes_FromStringAndSize( - (const char *)view.buf, view.len); - PyBuffer_Release(&view); - if (*cmd_copy == NULL) { - return NULL; - } - str = PyBytes_AS_STRING(*cmd_copy); - size = PyBytes_GET_SIZE(*cmd_copy); - } - else { - PyErr_Format(PyExc_TypeError, - "%s() arg 1 must be a %s object", - funcname, what); - return NULL; - } - - if (strlen(str) != (size_t)size) { - PyErr_SetString(PyExc_ValueError, - "source code string cannot contain null bytes"); - Py_CLEAR(*cmd_copy); - return NULL; - } - return str; -} - /*[clinic input] compile as builtin_compile @@ -855,7 +806,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto finally; } - str = source_as_string(source, "compile", "string, bytes or AST", &cf, &source_copy); + str = _Py_SourceAsString(source, "compile", "string, bytes or AST", &cf, &source_copy); if (str == NULL) goto error; @@ -987,7 +938,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, cf.cf_flags = PyCF_SOURCE_IS_UTF8; cf.cf_feature_version = PY_MINOR_VERSION; - str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy); + str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy); if (str == NULL) return NULL; @@ -1075,7 +1026,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyCompilerFlags cf; cf.cf_flags = PyCF_SOURCE_IS_UTF8; cf.cf_feature_version = PY_MINOR_VERSION; - str = source_as_string(source, "exec", + str = _Py_SourceAsString(source, "exec", "string, bytes or code", &cf, &source_copy); if (str == NULL) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index d2b27615903fee..b81b694fa5bb6f 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1184,21 +1184,77 @@ PyCompileString(const char *str, const char *filename, int start) return Py_CompileStringFlags(str, filename, start, NULL); } +const char * +_Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy) +{ + const char *str; + Py_ssize_t size; + Py_buffer view; + + *cmd_copy = NULL; + if (PyUnicode_Check(cmd)) { + cf->cf_flags |= PyCF_IGNORE_COOKIE; + str = PyUnicode_AsUTF8AndSize(cmd, &size); + if (str == NULL) + return NULL; + } + else if (PyBytes_Check(cmd)) { + str = PyBytes_AS_STRING(cmd); + size = PyBytes_GET_SIZE(cmd); + } + else if (PyByteArray_Check(cmd)) { + str = PyByteArray_AS_STRING(cmd); + size = PyByteArray_GET_SIZE(cmd); + } + else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) { + /* Copy to NUL-terminated buffer. */ + *cmd_copy = PyBytes_FromStringAndSize( + (const char *)view.buf, view.len); + PyBuffer_Release(&view); + if (*cmd_copy == NULL) { + return NULL; + } + str = PyBytes_AS_STRING(*cmd_copy); + size = PyBytes_GET_SIZE(*cmd_copy); + } + else { + PyErr_Format(PyExc_TypeError, + "%s() arg 1 must be a %s object", + funcname, what); + return NULL; + } + + if (strlen(str) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + Py_CLEAR(*cmd_copy); + return NULL; + } + return str; +} + struct symtable * Py_SymtableStringObject(const char *str, PyObject *filename, int start) +{ + PyCompilerFlags flags; + + flags.cf_flags = 0; + flags.cf_feature_version = PY_MINOR_VERSION; + return _Py_SymtableStringObjectFlags(str, filename, start, &flags); +} + +struct symtable * +_Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, PyCompilerFlags *flags) { struct symtable *st; mod_ty mod; - PyCompilerFlags flags; PyArena *arena; arena = PyArena_New(); if (arena == NULL) return NULL; - flags.cf_flags = 0; - flags.cf_feature_version = PY_MINOR_VERSION; - mod = PyParser_ASTFromStringObject(str, filename, start, &flags, arena); + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); if (mod == NULL) { PyArena_Free(arena); return NULL; From 25b7478f9859ccef061334d0cbbcdee51dab485d Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Thu, 23 May 2019 21:11:10 -0700 Subject: [PATCH 2/5] Add NEWS blurb --- .../next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst diff --git a/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst b/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst new file mode 100644 index 00000000000000..bb2ad74f790446 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst @@ -0,0 +1,2 @@ +symtable.symtable now accepts the same input types for source code as the +built-in compile function. From cc8450cc5b198810f5259e98048611c30569fe9d Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Mon, 27 May 2019 19:54:04 -0700 Subject: [PATCH 3/5] Ensure source_copy is always decref'd rename str->source - lso matching compile, after verifying we don't currently accept kw arguments --- Modules/symtablemodule.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index f93cfce902e107..affaca069d5a13 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -14,7 +14,7 @@ module _symtable /*[clinic input] _symtable.symtable - str: object + source: object filename: object(converter='PyUnicode_FSDecoder') startstr: str / @@ -23,7 +23,7 @@ Return symbol and scope dictionaries used internally by compiler. [clinic start generated code]*/ static PyObject * -_symtable_symtable_impl(PyObject *module, PyObject *str, PyObject *filename, +_symtable_symtable_impl(PyObject *module, PyObject *source, PyObject *filename, const char *startstr) /*[clinic end generated code: output=da9d42d2103ea619 input=a8b56a8eee46c26b]*/ { @@ -36,8 +36,8 @@ _symtable_symtable_impl(PyObject *module, PyObject *str, PyObject *filename, cf.cf_flags = PyCF_SOURCE_IS_UTF8; cf.cf_feature_version = PY_MINOR_VERSION; - const char *source = _Py_SourceAsString(str, "symtable", "string or bytes", &cf, &source_copy); - if (source == NULL) { + const char *str = _Py_SourceAsString(source, "symtable", "string or bytes", &cf, &source_copy); + if (str == NULL) { return NULL; } @@ -54,10 +54,10 @@ _symtable_symtable_impl(PyObject *module, PyObject *str, PyObject *filename, Py_XDECREF(source_copy); return NULL; } - st = _Py_SymtableStringObjectFlags(source, filename, start, &cf); + st = _Py_SymtableStringObjectFlags(str, filename, start, &cf); Py_DECREF(filename); + Py_XDECREF(source_copy); if (st == NULL) { - Py_XDECREF(source_copy); return NULL; } t = (PyObject *)st->st_top; From 540d802b6dad1294df75912ac7f0ddb58e4ece47 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 28 May 2019 11:36:26 +0300 Subject: [PATCH 4/5] Update 2019-05-23-21-10-57.bpo-37001.DoLvTK.rst --- .../next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst b/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst index bb2ad74f790446..5bcd7a9976c553 100644 --- a/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst +++ b/Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst @@ -1,2 +1,2 @@ -symtable.symtable now accepts the same input types for source code as the -built-in compile function. +:func:`symtable.symtable` now accepts the same input types for source code as the +built-in :func:`compile` function. Patch by Dino Viehland. From 9d380caa858d74f91e759f0be821a66b2f3e3554 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Tue, 28 May 2019 16:01:20 -0700 Subject: [PATCH 5/5] Fix clinic generated code --- Modules/clinic/symtablemodule.c.h | 14 +++++++------- Modules/symtablemodule.c | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index d6021e444fe8f1..7d8b0ad300c2cf 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -3,7 +3,7 @@ preserve [clinic start generated code]*/ PyDoc_STRVAR(_symtable_symtable__doc__, -"symtable($module, str, filename, startstr, /)\n" +"symtable($module, source, filename, startstr, /)\n" "--\n" "\n" "Return symbol and scope dictionaries used internally by compiler."); @@ -12,21 +12,21 @@ PyDoc_STRVAR(_symtable_symtable__doc__, {"symtable", (PyCFunction)(void(*)(void))_symtable_symtable, METH_FASTCALL, _symtable_symtable__doc__}, static PyObject * -_symtable_symtable_impl(PyObject *module, PyObject *str, PyObject *filename, - const char *startstr); +_symtable_symtable_impl(PyObject *module, PyObject *source, + PyObject *filename, const char *startstr); static PyObject * _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - PyObject *str; + PyObject *source; PyObject *filename; const char *startstr; if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { goto exit; } - str = args[0]; + source = args[0]; if (!PyUnicode_FSDecoder(args[1], &filename)) { goto exit; } @@ -43,9 +43,9 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _symtable_symtable_impl(module, str, filename, startstr); + return_value = _symtable_symtable_impl(module, source, filename, startstr); exit: return return_value; } -/*[clinic end generated code: output=a72eb8187a6bc8aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=de655625eee705f4 input=a9049054013a1b77]*/ diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index affaca069d5a13..d66cb44f69bdfe 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -23,9 +23,9 @@ Return symbol and scope dictionaries used internally by compiler. [clinic start generated code]*/ static PyObject * -_symtable_symtable_impl(PyObject *module, PyObject *source, PyObject *filename, - const char *startstr) -/*[clinic end generated code: output=da9d42d2103ea619 input=a8b56a8eee46c26b]*/ +_symtable_symtable_impl(PyObject *module, PyObject *source, + PyObject *filename, const char *startstr) +/*[clinic end generated code: output=59eb0d5fc7285ac4 input=9dd8a50c0c36a4d7]*/ { struct symtable *st; PyObject *t;