From 0eed7ba07dc5da2c591b484cd6f0172f5d4c81a5 Mon Sep 17 00:00:00 2001 From: Chris Trimble Date: Wed, 14 Aug 2019 13:18:32 -0700 Subject: [PATCH 1/2] Make C modules Python 3 compatible _baked.c was the most complicated one. Py3 Unicode implementation requires initialization. I left the Python 2 implementation alone because I'm not entirely sure of the overhead being saved with memcpy right now. --- spitfire/runtime/_baked.c | 100 +++++++++++++++++++++++++++++------ spitfire/runtime/_template.c | 55 ++++++++++++++----- spitfire/runtime/_udn.c | 33 ++++++++++-- 3 files changed, 155 insertions(+), 33 deletions(-) diff --git a/spitfire/runtime/_baked.c b/spitfire/runtime/_baked.c index 5813c63..33b1c1f 100644 --- a/spitfire/runtime/_baked.c +++ b/spitfire/runtime/_baked.c @@ -10,7 +10,12 @@ static PyObject *Skip_Filter_PyString = NULL; // SanitizedPlaceholder Object typedef struct { +#if PY_MAJOR_VERSION >= 3 + + PyUnicodeObject str; +#else PyStringObject str; +#endif } SanitizedPlaceholderObject; // Forward declare the type object. @@ -22,6 +27,17 @@ static PyTypeObject SanitizedPlaceholderType; static PyObject * mark_as_sanitized(PyObject *self, PyObject *value) { +#if PY_MAJOR_VERSION >= 3 + // If the value is not unicode, return it immediately. + if (!PyUnicode_CheckExact(value)) { + Py_INCREF(value); + return value; + } + PyObject *args = Py_BuildValue("(N)", value); + PyObject *obj = PyObject_CallObject((PyObject *) &SanitizedPlaceholderType, args); + Py_DECREF(args); + return obj; +#else // If the value is not a string, return it immediately. if (!PyString_CheckExact(value)) { Py_INCREF(value); @@ -31,6 +47,9 @@ mark_as_sanitized(PyObject *self, PyObject *value) Py_ssize_t size = PyString_GET_SIZE(value); // Allocate space for the SP object. PyObject *obj = PyType_GenericAlloc(&SanitizedPlaceholderType, size); + if (obj == NULL) { + return NULL; + } SanitizedPlaceholderObject *sp_ptr = (SanitizedPlaceholderObject *)obj; // Set the hash and state. sp_ptr->str.ob_shash = ((PyStringObject *)value)->ob_shash; @@ -39,6 +58,7 @@ mark_as_sanitized(PyObject *self, PyObject *value) // instantiating a new string object. memcpy(sp_ptr->str.ob_sval, PyString_AS_STRING(value), size+1); return obj; +#endif } // Python function that checks if skip_filter is present on the function passed @@ -71,15 +91,21 @@ runtime_mark_as_sanitized(PyObject *self, PyObject *args) static PyObject * sp_concat(PyObject *self, PyObject *other) { + PyObject *tmpself; +#if PY_MAJOR_VERSION >= 3 + tmpself = PyUnicode_Concat(self, other); +#else // PyString_Concat requires an INCREF on self. Py_INCREF(self); PyString_Concat(&self, other); + tmpself = self; +#endif if (Py_TYPE(other) != &SanitizedPlaceholderType) { - return self; + return tmpself; } // PyString_Concat copies self turning it back into a string. Calling // mark_as_sanitized turns it back into a SanitizedPlaceholder. - return mark_as_sanitized(self, self); + return mark_as_sanitized(tmpself, tmpself); } // SanitizedPlaceholder method for mod (%). A SanitizedPlaceholder is returned @@ -87,12 +113,16 @@ sp_concat(PyObject *self, PyObject *other) static PyObject * sp_mod(PyObject *self, PyObject *other) { - PyObject * val = PyString_Format(self, other); +#if PY_MAJOR_VERSION >= 3 + PyObject *val = PyUnicode_Format(self, other); +#else + PyObject *val = PyString_Format(self, other); +#endif if (Py_TYPE(other) != &SanitizedPlaceholderType) { return val; } // If the other value was a SanitizedPlaceholder, we want to turn the string - // result from PyString_Format into a SanitizedPlaceholder. This currently + // result from Py*_Format into a SanitizedPlaceholder. This currently // copies over the string which is not the most efficient way to do this. return mark_as_sanitized(self, val); } @@ -112,14 +142,15 @@ static PyNumberMethods sp_as_number = { 0, /* nb_add */ 0, /* nb_subtract */ 0, /* nb_multiply */ +#if PY_MAJOR_VERSION < 3 0, /* nb_divide */ +#endif sp_mod, /* nb_remainder */ }; // SanitizedPlaceholder Type. static PyTypeObject SanitizedPlaceholderType = { - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ + PyVarObject_HEAD_INIT(NULL,0 ) "baked.SanitizedPlaceholder", /* tp_name */ sizeof(SanitizedPlaceholderObject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -139,8 +170,12 @@ static PyTypeObject SanitizedPlaceholderType = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_CHECKTYPES | - Py_TPFLAGS_STRING_SUBCLASS, /* tp_flags */ +#if PY_MAJOR_VERSION >= 3 + Py_TPFLAGS_UNICODE_SUBCLASS, /* tp_flags */ +#else + Py_TPFLAGS_CHECKTYPES | + Py_TPFLAGS_STRING_SUBCLASS, /* tp_flags */ +#endif 0, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ @@ -151,7 +186,7 @@ static PyTypeObject SanitizedPlaceholderType = { 0, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ - 0, /* tp_base */ + &PyUnicode_Type, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -169,25 +204,58 @@ static struct PyMethodDef baked_functions[] = { {NULL, NULL} }; -PyMODINIT_FUNC -init_baked(void) +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_baked", /* m_name */ + "Baked Module", /* m_doc */ + -1, /* m_size */ + baked_functions, /* m_methods */ + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL, /* m_free */ +}; +#endif + +PyObject* moduleinit(void) { PyObject *m; +#if PY_MAJOR_VERSION >= 3 + Skip_Filter_PyString = PyUnicode_InternFromString("skip_filter"); + // Set the base type and the constructor. + SanitizedPlaceholderType.tp_base = &PyUnicode_Type; + SanitizedPlaceholderType.tp_init = PyUnicode_Type.tp_init; +#else Skip_Filter_PyString = PyString_InternFromString("skip_filter"); - // Set the base type and the constructor. SanitizedPlaceholderType.tp_base = &PyString_Type; SanitizedPlaceholderType.tp_init = PyString_Type.tp_init; - if (PyType_Ready(&SanitizedPlaceholderType) < 0) - return; +#endif + + if (PyType_Ready(&SanitizedPlaceholderType) < 0) { + return NULL; + } // Add exported functions to the module. +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&moduledef); +#else m = Py_InitModule3("_baked", baked_functions, "Baked Module"); - if (m == NULL) - return; +#endif + if (m == NULL) { + return NULL; + } Py_INCREF(&SanitizedPlaceholderType); PyModule_AddObject(m, "_SanitizedPlaceholder", (PyObject *) &SanitizedPlaceholderType); + return m; } + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC init_baked(void) { (void)moduleinit(); } +#else +PyMODINIT_FUNC PyInit__baked(void) { return moduleinit(); } +#endif diff --git a/spitfire/runtime/_template.c b/spitfire/runtime/_template.c index 31b9000..f4426bb 100644 --- a/spitfire/runtime/_template.c +++ b/spitfire/runtime/_template.c @@ -74,8 +74,7 @@ static struct PyMethodDef template_methods[] = { // BaseSpitfireTemplate Type. static PyTypeObject BaseSpitfireTemplateType = { - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ + PyVarObject_HEAD_INIT(NULL, 0) "template.BaseSpitfireTemplate", /* tp_name */ sizeof(BaseSpitfireTemplateObject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -121,36 +120,68 @@ static struct PyMethodDef module_methods[] = { {NULL} // Sentinel }; +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_template", /* m_name */ + "Template Module", /* m_doc */ + -1, /* m_size */ + module_methods, /* m_methods */ + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL, /* m_free */ +}; +#endif -PyMODINIT_FUNC -init_template(void) +PyObject* moduleinit(void) { // Set interned strings. +#if PY_MAJOR_VERSION >= 3 + Skip_Filter_PyString = PyUnicode_InternFromString("skip_filter"); + filter_function_name = PyUnicode_InternFromString("_filter_function"); +#else Skip_Filter_PyString = PyString_InternFromString("skip_filter"); filter_function_name = PyString_InternFromString("_filter_function"); +#endif // Get SanitizedPlaceholder from the baked module. PyObject *baked_module = PyImport_ImportModule("spitfire.runtime.baked"); - if (baked_module == NULL) - return; + if (baked_module == NULL) { + return NULL; + } baked_SanitizedPlaceholder = (struct _typeobject *) PyObject_GetAttrString(baked_module, "SanitizedPlaceholder"); Py_DECREF(baked_module); - if (baked_SanitizedPlaceholder == NULL) - return; + if (baked_SanitizedPlaceholder == NULL) { + return NULL; + } // Setup module and class. PyObject *m; BaseSpitfireTemplateType.tp_new = PyType_GenericNew; - if (PyType_Ready(&BaseSpitfireTemplateType) < 0) - return; + if (PyType_Ready(&BaseSpitfireTemplateType) < 0) { + return NULL; + } +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&moduledef); +#else m = Py_InitModule3("_template", module_methods, "Template Module"); - if (m == NULL) - return; +#endif + if (m == NULL) { + return NULL; + } Py_INCREF(&BaseSpitfireTemplateType); PyModule_AddObject(m, "BaseSpitfireTemplate", (PyObject *)&BaseSpitfireTemplateType); + return m; } + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC init_template(void) { (void)moduleinit(); } +#else +PyMODINIT_FUNC PyInit__template(void) { return moduleinit(); } +#endif diff --git a/spitfire/runtime/_udn.c b/spitfire/runtime/_udn.c index c153a48..911b142 100644 --- a/spitfire/runtime/_udn.c +++ b/spitfire/runtime/_udn.c @@ -72,7 +72,7 @@ udn_resolve_udn(PyObject *self, PyObject *args, PyObject *kargs) return NULL; } - if (!(PyUnicode_Check(name) || PyString_Check(name))) { + if (!(PyUnicode_Check(name) || PyBytes_Check(name))) { PyErr_SetString(PyExc_ValueError, "name must be string"); return NULL; } @@ -110,7 +110,7 @@ udn_resolve_from_search_list(PyObject *self, PyObject *args, PyObject *keywds) return NULL; } - if (!(PyUnicode_Check(name) || PyString_Check(name))) { + if (!(PyUnicode_Check(name) || PyBytes_Check(name))) { PyErr_SetString(PyExc_ValueError, "name must be string"); return NULL; } @@ -155,15 +155,31 @@ static struct PyMethodDef udn_methods[] = { {NULL, NULL} }; +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "_udn", /* m_name */ + "_udn module", /* m_doc */ + -1, /* m_size */ + udn_methods, /* m_methods */ + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL, /* m_free */ +}; +#endif /* Initialization function (import-time) */ -DL_EXPORT(void) -init_udn(void) +static PyObject* moduleinit(void) { PyObject *m, *runtime_module; - m = Py_InitModule("_udn", udn_methods); +#if PY_MAJOR_VERSION >= 3 + m = PyModule_Create(&moduledef); +#else + m = Py_InitModule3("_udn", udn_methods, "_udn module"); +#endif runtime_module = PyImport_ImportModule("spitfire.runtime"); PlaceholderError = PyObject_GetAttrString( @@ -177,8 +193,15 @@ init_udn(void) if (PyErr_Occurred()) Py_FatalError("Can't initialize module _udn"); + return m; } +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC init_udn(void) { (void)moduleinit(); } +#else +PyMODINIT_FUNC PyInit__udn(void) { return moduleinit(); } +#endif + #ifdef __cplusplus } #endif From 36fdb4110e5f57ebb1c3742d8bb12711b258ed1b Mon Sep 17 00:00:00 2001 From: Chris Trimble Date: Wed, 28 Aug 2019 15:09:58 -0700 Subject: [PATCH 2/2] Update spitfire/runtime/_baked.c Co-Authored-By: Alex Nicksay --- spitfire/runtime/_baked.c | 1 - 1 file changed, 1 deletion(-) diff --git a/spitfire/runtime/_baked.c b/spitfire/runtime/_baked.c index 33b1c1f..b1990ee 100644 --- a/spitfire/runtime/_baked.c +++ b/spitfire/runtime/_baked.c @@ -11,7 +11,6 @@ static PyObject *Skip_Filter_PyString = NULL; // SanitizedPlaceholder Object typedef struct { #if PY_MAJOR_VERSION >= 3 - PyUnicodeObject str; #else PyStringObject str;