Skip to content

Commit 4154069

Browse files
DinoVDinoV
authored
bpo-37001: Makes symtable.symtable have parity with compile for input (#13483)
* Makes symtable.symtable have parity for accepted datatypes for source code as compile() * Add NEWS blurb
1 parent ab0716e commit 4154069

File tree

7 files changed

+111
-79
lines changed

7 files changed

+111
-79
lines changed

‎Include/pythonrun.h

+13
Original file line numberDiff line numberDiff line change
@@ -119,10 +119,23 @@ PyAPI_FUNC(struct symtable *) Py_SymtableString(
119119
const char *filename, /* decoded from the filesystem encoding */
120120
int start);
121121
#ifndef Py_LIMITED_API
122+
PyAPI_FUNC(const char *) _Py_SourceAsString(
123+
PyObject *cmd,
124+
const char *funcname,
125+
const char *what,
126+
PyCompilerFlags *cf,
127+
PyObject **cmd_copy);
128+
122129
PyAPI_FUNC(struct symtable *) Py_SymtableStringObject(
123130
const char *str,
124131
PyObject *filename,
125132
int start);
133+
134+
PyAPI_FUNC(struct symtable *) _Py_SymtableStringObjectFlags(
135+
const char *str,
136+
PyObject *filename,
137+
int start,
138+
PyCompilerFlags *flags);
126139
#endif
127140

128141
PyAPI_FUNC(void) PyErr_Print(void);

‎Lib/test/test_symtable.py

+9
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,15 @@ def test_single(self):
215215
def test_exec(self):
216216
symbols = symtable.symtable("def f(x): return x", "?", "exec")
217217

218+
def test_bytes(self):
219+
top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec")
220+
self.assertIsNotNone(find_block(top, "Mine"))
221+
222+
code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n'
223+
224+
top = symtable.symtable(code, "?", "exec")
225+
self.assertIsNotNone(find_block(top, "\u017d"))
226+
218227

219228
if __name__ == '__main__':
220229
unittest.main()

‎Misc/NEWS.d/next/Library/2019-05-23-21-10-57.bpo-37001.DoLvTK.rst

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
:func:`symtable.symtable` now accepts the same input types for source code as the
2+
built-in :func:`compile` function. Patch by Dino Viehland.

‎Modules/clinic/symtablemodule.c.h

+6-18
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Modules/symtablemodule.c

+18-5
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ module _symtable
1414
/*[clinic input]
1515
_symtable.symtable
1616
17-
str: str
17+
source: object
1818
filename: object(converter='PyUnicode_FSDecoder')
1919
startstr: str
2020
/
@@ -23,13 +23,23 @@ Return symbol and scope dictionaries used internally by compiler.
2323
[clinic start generated code]*/
2424

2525
static PyObject *
26-
_symtable_symtable_impl(PyObject *module, const char *str,
26+
_symtable_symtable_impl(PyObject *module, PyObject *source,
2727
PyObject *filename, const char *startstr)
28-
/*[clinic end generated code: output=914b369c9b785956 input=6c615e84d5f408e3]*/
28+
/*[clinic end generated code: output=59eb0d5fc7285ac4 input=9dd8a50c0c36a4d7]*/
2929
{
3030
struct symtable *st;
3131
PyObject *t;
3232
int start;
33+
PyCompilerFlags cf;
34+
PyObject *source_copy = NULL;
35+
36+
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
37+
cf.cf_feature_version = PY_MINOR_VERSION;
38+
39+
const char *str = _Py_SourceAsString(source, "symtable", "string or bytes", &cf, &source_copy);
40+
if (str == NULL) {
41+
return NULL;
42+
}
3343

3444
if (strcmp(startstr, "exec") == 0)
3545
start = Py_file_input;
@@ -41,12 +51,15 @@ _symtable_symtable_impl(PyObject *module, const char *str,
4151
PyErr_SetString(PyExc_ValueError,
4252
"symtable() arg 3 must be 'exec' or 'eval' or 'single'");
4353
Py_DECREF(filename);
54+
Py_XDECREF(source_copy);
4455
return NULL;
4556
}
46-
st = Py_SymtableStringObject(str, filename, start);
57+
st = _Py_SymtableStringObjectFlags(str, filename, start, &cf);
4758
Py_DECREF(filename);
48-
if (st == NULL)
59+
Py_XDECREF(source_copy);
60+
if (st == NULL) {
4961
return NULL;
62+
}
5063
t = (PyObject *)st->st_top;
5164
Py_INCREF(t);
5265
PyMem_Free((void *)st->st_future);

‎Python/bltinmodule.c

+3-52
Original file line numberDiff line numberDiff line change
@@ -687,55 +687,6 @@ builtin_chr_impl(PyObject *module, int i)
687687
}
688688

689689

690-
static const char *
691-
source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
692-
{
693-
const char *str;
694-
Py_ssize_t size;
695-
Py_buffer view;
696-
697-
*cmd_copy = NULL;
698-
if (PyUnicode_Check(cmd)) {
699-
cf->cf_flags |= PyCF_IGNORE_COOKIE;
700-
str = PyUnicode_AsUTF8AndSize(cmd, &size);
701-
if (str == NULL)
702-
return NULL;
703-
}
704-
else if (PyBytes_Check(cmd)) {
705-
str = PyBytes_AS_STRING(cmd);
706-
size = PyBytes_GET_SIZE(cmd);
707-
}
708-
else if (PyByteArray_Check(cmd)) {
709-
str = PyByteArray_AS_STRING(cmd);
710-
size = PyByteArray_GET_SIZE(cmd);
711-
}
712-
else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
713-
/* Copy to NUL-terminated buffer. */
714-
*cmd_copy = PyBytes_FromStringAndSize(
715-
(const char *)view.buf, view.len);
716-
PyBuffer_Release(&view);
717-
if (*cmd_copy == NULL) {
718-
return NULL;
719-
}
720-
str = PyBytes_AS_STRING(*cmd_copy);
721-
size = PyBytes_GET_SIZE(*cmd_copy);
722-
}
723-
else {
724-
PyErr_Format(PyExc_TypeError,
725-
"%s() arg 1 must be a %s object",
726-
funcname, what);
727-
return NULL;
728-
}
729-
730-
if (strlen(str) != (size_t)size) {
731-
PyErr_SetString(PyExc_ValueError,
732-
"source code string cannot contain null bytes");
733-
Py_CLEAR(*cmd_copy);
734-
return NULL;
735-
}
736-
return str;
737-
}
738-
739690
/*[clinic input]
740691
compile as builtin_compile
741692
@@ -855,7 +806,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
855806
goto finally;
856807
}
857808

858-
str = source_as_string(source, "compile", "string, bytes or AST", &cf, &source_copy);
809+
str = _Py_SourceAsString(source, "compile", "string, bytes or AST", &cf, &source_copy);
859810
if (str == NULL)
860811
goto error;
861812

@@ -991,7 +942,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals,
991942

992943
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
993944
cf.cf_feature_version = PY_MINOR_VERSION;
994-
str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy);
945+
str = _Py_SourceAsString(source, "eval", "string, bytes or code", &cf, &source_copy);
995946
if (str == NULL)
996947
return NULL;
997948

@@ -1083,7 +1034,7 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals,
10831034
PyCompilerFlags cf;
10841035
cf.cf_flags = PyCF_SOURCE_IS_UTF8;
10851036
cf.cf_feature_version = PY_MINOR_VERSION;
1086-
str = source_as_string(source, "exec",
1037+
str = _Py_SourceAsString(source, "exec",
10871038
"string, bytes or code", &cf,
10881039
&source_copy);
10891040
if (str == NULL)

‎Python/pythonrun.c

+60-4
Original file line numberDiff line numberDiff line change
@@ -1231,21 +1231,77 @@ PyCompileString(const char *str, const char *filename, int start)
12311231
return Py_CompileStringFlags(str, filename, start, NULL);
12321232
}
12331233

1234+
const char *
1235+
_Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy)
1236+
{
1237+
const char *str;
1238+
Py_ssize_t size;
1239+
Py_buffer view;
1240+
1241+
*cmd_copy = NULL;
1242+
if (PyUnicode_Check(cmd)) {
1243+
cf->cf_flags |= PyCF_IGNORE_COOKIE;
1244+
str = PyUnicode_AsUTF8AndSize(cmd, &size);
1245+
if (str == NULL)
1246+
return NULL;
1247+
}
1248+
else if (PyBytes_Check(cmd)) {
1249+
str = PyBytes_AS_STRING(cmd);
1250+
size = PyBytes_GET_SIZE(cmd);
1251+
}
1252+
else if (PyByteArray_Check(cmd)) {
1253+
str = PyByteArray_AS_STRING(cmd);
1254+
size = PyByteArray_GET_SIZE(cmd);
1255+
}
1256+
else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {
1257+
/* Copy to NUL-terminated buffer. */
1258+
*cmd_copy = PyBytes_FromStringAndSize(
1259+
(const char *)view.buf, view.len);
1260+
PyBuffer_Release(&view);
1261+
if (*cmd_copy == NULL) {
1262+
return NULL;
1263+
}
1264+
str = PyBytes_AS_STRING(*cmd_copy);
1265+
size = PyBytes_GET_SIZE(*cmd_copy);
1266+
}
1267+
else {
1268+
PyErr_Format(PyExc_TypeError,
1269+
"%s() arg 1 must be a %s object",
1270+
funcname, what);
1271+
return NULL;
1272+
}
1273+
1274+
if (strlen(str) != (size_t)size) {
1275+
PyErr_SetString(PyExc_ValueError,
1276+
"source code string cannot contain null bytes");
1277+
Py_CLEAR(*cmd_copy);
1278+
return NULL;
1279+
}
1280+
return str;
1281+
}
1282+
12341283
struct symtable *
12351284
Py_SymtableStringObject(const char *str, PyObject *filename, int start)
1285+
{
1286+
PyCompilerFlags flags;
1287+
1288+
flags.cf_flags = 0;
1289+
flags.cf_feature_version = PY_MINOR_VERSION;
1290+
return _Py_SymtableStringObjectFlags(str, filename, start, &flags);
1291+
}
1292+
1293+
struct symtable *
1294+
_Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, PyCompilerFlags *flags)
12361295
{
12371296
struct symtable *st;
12381297
mod_ty mod;
1239-
PyCompilerFlags flags;
12401298
PyArena *arena;
12411299

12421300
arena = PyArena_New();
12431301
if (arena == NULL)
12441302
return NULL;
12451303

1246-
flags.cf_flags = 0;
1247-
flags.cf_feature_version = PY_MINOR_VERSION;
1248-
mod = PyParser_ASTFromStringObject(str, filename, start, &flags, arena);
1304+
mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
12491305
if (mod == NULL) {
12501306
PyArena_Free(arena);
12511307
return NULL;

0 commit comments

Comments
 (0)