Use Py_LIMITED_API

bin/build-sip-msw

@@ -22,9 +22,7 @@ MYBINDIR=$(dirname $(readlink -f $0))
 cd /c/projects/sip/sip
 SIPVER=$($PYTHON -c "import sys,configure; sys.stdout.write(configure.sip_version_str);")
 
-$PYTHON configure.py \
-    --sip-module wx.siplib \
-    $*
+$PYTHON configure.py $*
 
 cd sipgen
 nmake clean all

bin/build-sip-posix

@@ -36,15 +36,12 @@ if [ "$PLATFORM" = "darwin" ]; then
 	--sdk=$SDK \
 	--arch=i386 \
 	--universal \
-	--sip-module wx.siplib \
 	$*
 
     make -C sipgen clean all
 
 else
-    $PYTHON configure.py \
-	--sip-module wx.siplib \
-	$*
+    $PYTHON configure.py $*
 
     make -C sipgen clean all
 fi

build.py

@@ -391,7 +391,11 @@ def makeOptionParser():
         ("debug",          (False, "Build wxPython with debug symbols")),
         ("relwithdebug",   (False, "Turn on the generation of debug info for release builds on MSW.")),
         ("release",        (False, "Turn off some development options for a release build.")),
+
+        ("py_limited_api", (False, "Set flags to use the Python Limited API.")),
+
         ("keep_hash_lines",(False, "Don't remove the '#line N' lines from the SIP generated code")),
+
         ("gtk3",           (False, "On Linux build for gtk3 (default gtk2)")),
         ("osx_cocoa",      (True,  "Build the OSX Cocoa port on Mac (default)")),
         ("osx_carbon",     (False, "Build the OSX Carbon port on Mac (unsupported)")),
@@ -1398,6 +1402,9 @@ def cmd_build_py(options, args):
     if options.gtk3:
         build_options.append('--gtk3')
 
+    if options.py_limited_api:
+        build_options.append('--py_limited_api')
+
     build_options.append('--python="%s"' % PYTHON)
     build_options.append('--out=%s' % wafBuildDir) # this needs to be the last option
 

buildtools/config.py

@@ -111,6 +111,7 @@ def __init__(self, noWxConfig=False):
 
         self.includes = [phoenixDir() + '/sip/siplib',  # to get our version of sip.h
                          phoenixDir() + '/src',         # for any hand-written headers
+                         phoenixDir() + '/wxpy_api',    # for our internal API module
                          ]
 
         self.DOXY_XML_DIR = os.path.join(self.WXDIR, 'docs/doxygen/out/xml')
@@ -541,7 +542,6 @@ def adjustCFLAGS(self, cflags, defines, includes):
         return newCFLAGS
 
 
-
     def adjustLFLAGS(self, lflags, libdirs, libs):
         """
         Extract the -L and -l flags from lflags and put them in libdirs and
@@ -558,6 +558,20 @@ def adjustLFLAGS(self, lflags, libdirs, libs):
         return newLFLAGS
 
 
+    def set_limited_api(self, use_limited_api):
+        """
+        Add a define for Py_LIMITED_API if the option is enabled
+        """
+        name = 'Py_LIMITED_API'
+        version = '0x03040000'
+        if use_limited_api:
+            # not sure yet where it needs to go, so put it everywhere ;-)
+            self.defines.append( (name, version) )
+            self.wafDefines.append('{}={}'.format(name, version))
+            # and return it too
+            return '-D{}={}'.format(name, version)
+
+
 
 # We'll use a factory function so we can use the Configuration class as a singleton
 _config = None

etg/_core.py

@@ -39,7 +39,6 @@
               'defs',
               'debug',
               'object',
-              'wxpy_api',
               'arrayholder',
               'string',
               'filename',
@@ -68,7 +67,8 @@
               'position',
               'colour',
 
-              'stream', 'filesys',
+              'stream',
+              'filesys',
 
               # GDI and graphics
               'image',

etg/accel.py

@@ -69,17 +69,17 @@ def run():
     }
     int idx;
     for (idx=0; idx<count; idx++) {
-        PyObject* obj = PySequence_ITEM(entries, idx);
+        PyObject* obj = PySequence_GetItem(entries, idx);
         if (sipCanConvertToType(obj, sipType_wxAcceleratorEntry, SIP_NO_CONVERTORS)) {
             int err = 0;
             wxAcceleratorEntry* entryPtr = reinterpret_cast<wxAcceleratorEntry*>(
                 sipConvertToType(obj, sipType_wxAcceleratorEntry, NULL, 0, 0, &err));
             tmpEntries[idx] = *entryPtr;
         }
         else if (PySequence_Check(obj) && PySequence_Size(obj) == 3) {
-            PyObject* o1 = PySequence_ITEM(obj, 0);
-            PyObject* o2 = PySequence_ITEM(obj, 1);
-            PyObject* o3 = PySequence_ITEM(obj, 2);
+            PyObject* o1 = PySequence_GetItem(obj, 0);
+            PyObject* o2 = PySequence_GetItem(obj, 1);
+            PyObject* o3 = PySequence_GetItem(obj, 2);
             tmpEntries[idx].Set(wxPyInt_AsLong(o1), wxPyInt_AsLong(o2), wxPyInt_AsLong(o3));
             Py_DECREF(o1);
             Py_DECREF(o2);

etg/bitmap.py

@@ -69,7 +69,7 @@ def run():
             cArray = new char*[count];
 
             for(int x=0; x<count; x++) {
-                PyObject* item = PyList_GET_ITEM(listOfBytes, x);
+                PyObject* item = PyList_GetItem(listOfBytes, x);
                 if (!PyBytes_Check(item)) {
                     PyErr_SetString(PyExc_TypeError, errMsg);
                     delete [] cArray;

etg/cmndata.py

@@ -83,7 +83,7 @@ def run():
             return;
         }
 
-        self->SetPrivData(PyBytes_AS_STRING(data), PyBytes_GET_SIZE(data));
+        self->SetPrivData(PyBytes_AsString(data), PyBytes_Size(data));
         """)
 
     c.addAutoProperties()

etg/colour.py

@@ -250,13 +250,13 @@ def run():
         else if (wxPyNumberSequenceCheck(sipPy)) {
             size_t len = PySequence_Size(sipPy);
 
-            PyObject* o1 = PySequence_ITEM(sipPy, 0);
-            PyObject* o2 = PySequence_ITEM(sipPy, 1);
-            PyObject* o3 = PySequence_ITEM(sipPy, 2);
+            PyObject* o1 = PySequence_GetItem(sipPy, 0);
+            PyObject* o2 = PySequence_GetItem(sipPy, 1);
+            PyObject* o3 = PySequence_GetItem(sipPy, 2);
             if (len == 3)
                 *sipCppPtr = new wxColour(wxPyInt_AsLong(o1), wxPyInt_AsLong(o2), wxPyInt_AsLong(o3));
             else {
-                PyObject* o4 = PySequence_ITEM(sipPy, 3);
+                PyObject* o4 = PySequence_GetItem(sipPy, 3);
                 *sipCppPtr = new wxColour(wxPyInt_AsLong(o1), wxPyInt_AsLong(o2), wxPyInt_AsLong(o3),
                                           wxPyInt_AsLong(o4));
                 Py_DECREF(o4);            

etg/config.py

@@ -96,9 +96,9 @@ def run():
         wxPyThreadBlocker blocker;
             PyObject* ret = PyTuple_New(3);
             if (ret) {
-                PyTuple_SET_ITEM(ret, 0, PyBool_FromLong(flag));
-                PyTuple_SET_ITEM(ret, 1, wx2PyString(str));
-                PyTuple_SET_ITEM(ret, 2, wxPyInt_FromLong(index));
+                PyTuple_SetItem(ret, 0, PyBool_FromLong(flag));
+                PyTuple_SetItem(ret, 1, wx2PyString(str));
+                PyTuple_SetItem(ret, 2, wxPyInt_FromLong(index));
             }
             return ret;
         }

etg/dataobj.py

@@ -60,7 +60,8 @@ def addGetAllFormats(klass, pureVirtual=False):
             for (size_t i=0; i<count; i++) {
                 wxDataFormat* format = new wxDataFormat(formats[i]);
                 PyObject* obj = wxPyConstructObject((void*)format, wxT("wxDataFormat"), true);
-                PyList_SET_ITEM(list, i, obj); // PyList_SET_ITEM steals a reference
+                PyList_SetItem(list, i, obj); 
+                // PyList_SetItem borrows the reference, so no need to decref it
             }
             delete [] formats;
             return list;

etg/dataview.py

@@ -417,8 +417,8 @@ def _fixupBoolGetters(method, sig):
                 col_obj = Py_None;
                 Py_INCREF(Py_None);
             }
-            PyTuple_SET_ITEM(value, 0, item_obj);
-            PyTuple_SET_ITEM(value, 1, col_obj);
+            PyTuple_SetItem(value, 0, item_obj);
+            PyTuple_SetItem(value, 1, col_obj);
             // PyTuple steals a reference, so we don't need to decref the items here
             return value;
             """)

etg/event.py

@@ -513,7 +513,7 @@ def run():
         }
         for (int i=0; i<count; i++) {
             PyObject* s = wx2PyString(files[i]);
-            PyList_SET_ITEM(list, i, s);
+            PyList_SetItem(list, i, s);
         }
         return list;
         """)

etg/palette.py

@@ -60,9 +60,9 @@ def run():
             unsigned char* blueArray = new unsigned char[count];
 
             for (Py_ssize_t i = 0; i < count; i++) {
-                PyObject* redItem = PySequence_ITEM(red, i);
-                PyObject* greenItem = PySequence_ITEM(green, i);
-                PyObject* blueItem = PySequence_ITEM(blue, i);
+                PyObject* redItem = PySequence_GetItem(red, i);
+                PyObject* greenItem = PySequence_GetItem(green, i);
+                PyObject* blueItem = PySequence_GetItem(blue, i);
                 if (!wxPyInt_Check(redItem) || !wxPyInt_Check(greenItem) || !wxPyInt_Check(blueItem)) {
                     PyErr_SetString(PyExc_TypeError, errMsg);
                     goto pch_exit;

etg/stream.py

@@ -274,7 +274,7 @@ def run():
             PyErr_SetString(PyExc_TypeError, "Bytes object expected");
             return NULL;
         }
-        self->Write(PyBytes_AS_STRING(data), PyBytes_GET_SIZE(data));
+        self->Write(PyBytes_AsString(data), PyBytes_Size(data));
         RETURN_NONE();
         """)
 

etg/treelist.py

@@ -96,7 +96,7 @@ def run():
             for (size_t i=0; i<count; i++) {
                 wxTreeListItem* item = new wxTreeListItem(items[i]);
                 PyObject* obj = wxPyConstructObject((void*)item, wxT("wxTreeListItem"), true);
-                PyList_SET_ITEM(list, i, obj); // PyList_SET_ITEM steals a reference
+                PyList_SetItem(list, i, obj); // steals a reference 
             }
             return list;
         """)

etgtools/tweaker_tools.py

@@ -679,8 +679,8 @@ def convertTwoIntegersTemplate(CLASS):
     }}
 
     // or create a new instance
-    PyObject* o1 = PySequence_ITEM(sipPy, 0);
-    PyObject* o2 = PySequence_ITEM(sipPy, 1);
+    PyObject* o1 = PySequence_GetItem(sipPy, 0);
+    PyObject* o2 = PySequence_GetItem(sipPy, 1);
     *sipCppPtr = new {CLASS}(wxPyInt_AsLong(o1), wxPyInt_AsLong(o2));
     Py_DECREF(o1);
     Py_DECREF(o2);
@@ -711,10 +711,10 @@ def convertFourIntegersTemplate(CLASS):
         return 0; // not a new instance
     }}
     // or create a new instance
-    PyObject* o1 = PySequence_ITEM(sipPy, 0);
-    PyObject* o2 = PySequence_ITEM(sipPy, 1);
-    PyObject* o3 = PySequence_ITEM(sipPy, 2);
-    PyObject* o4 = PySequence_ITEM(sipPy, 3);
+    PyObject* o1 = PySequence_GetItem(sipPy, 0);
+    PyObject* o2 = PySequence_GetItem(sipPy, 1);
+    PyObject* o3 = PySequence_GetItem(sipPy, 2);
+    PyObject* o4 = PySequence_GetItem(sipPy, 3);
     *sipCppPtr = new {CLASS}(wxPyInt_AsLong(o1), wxPyInt_AsLong(o2),
                              wxPyInt_AsLong(o3), wxPyInt_AsLong(o4));
     Py_DECREF(o1);
@@ -750,8 +750,8 @@ def convertTwoDoublesTemplate(CLASS):
     }}
 
     // or create a new instance
-    PyObject* o1 = PySequence_ITEM(sipPy, 0);
-    PyObject* o2 = PySequence_ITEM(sipPy, 1);
+    PyObject* o1 = PySequence_GetItem(sipPy, 0);
+    PyObject* o2 = PySequence_GetItem(sipPy, 1);
     *sipCppPtr = new {CLASS}(PyFloat_AsDouble(o1), PyFloat_AsDouble(o2));
     Py_DECREF(o1);
     Py_DECREF(o2);
@@ -783,10 +783,10 @@ def convertFourDoublesTemplate(CLASS):
     }}
 
     // or create a new instance
-    PyObject* o1 = PySequence_ITEM(sipPy, 0);
-    PyObject* o2 = PySequence_ITEM(sipPy, 1);
-    PyObject* o3 = PySequence_ITEM(sipPy, 2);
-    PyObject* o4 = PySequence_ITEM(sipPy, 3);
+    PyObject* o1 = PySequence_GetItem(sipPy, 0);
+    PyObject* o2 = PySequence_GetItem(sipPy, 1);
+    PyObject* o3 = PySequence_GetItem(sipPy, 2);
+    PyObject* o4 = PySequence_GetItem(sipPy, 3);
     *sipCppPtr = new {CLASS}(PyFloat_AsDouble(o1), PyFloat_AsDouble(o2),
                              PyFloat_AsDouble(o3), PyFloat_AsDouble(o4));
     Py_DECREF(o1);
@@ -932,7 +932,7 @@ def _{ListClass_pyName}___repr__(self):
         else {{
             Py_ssize_t i, len = PySequence_Length(sipPy);
             for (i=0; i<len; i++) {{
-                PyObject* item = PySequence_ITEM(sipPy, i);
+                PyObject* item = PySequence_GetItem(sipPy, i);
                 if (!sipCanConvertToType(item, sipType_{ItemClass}, SIP_NOT_NONE)) {{
                     Py_DECREF(item);
                     success = FALSE;
@@ -960,7 +960,7 @@ def _{ListClass_pyName}___repr__(self):
     Py_ssize_t i, len = PySequence_Length(sipPy);
     for (i=0; i<len; i++) {{
         int state;
-        PyObject* pyItem = PySequence_ITEM(sipPy, i);
+        PyObject* pyItem = PySequence_GetItem(sipPy, i);
         {ItemClass}* cItem = reinterpret_cast<{ItemClass}*>(
                              sipConvertToType(pyItem, sipType_{ItemClass},
                              NULL, 0, &state, sipIsErr));
@@ -1145,7 +1145,7 @@ def ObjArrayHelperTemplate(objType, sipType, errmsg):
     else {{
         len = PySequence_Length(source);
         for (idx=0; idx<len; idx++) {{
-            PyObject* item = PySequence_ITEM(source, idx);
+            PyObject* item = PySequence_GetItem(source, idx);
             if (!sipCanConvertToType(item, {sipType}, SIP_NOT_NONE)) {{
                 Py_DECREF(item);
                 goto error0;
@@ -1162,7 +1162,7 @@ def ObjArrayHelperTemplate(objType, sipType, errmsg):
         return NULL;
     }}
     for (idx=0; idx<len; idx++) {{
-        PyObject* obj = PySequence_ITEM(source, idx);
+        PyObject* obj = PySequence_GetItem(source, idx);
         int state = 0;
         int err = 0;
         {objType}* item = reinterpret_cast<{objType}*>(

py_limited_api.txt

@@ -0,0 +1,70 @@
+
+
+-------------------------------------------------
+Temporary notes, do not commit in the final merge
+-------------------------------------------------
+
+
+New bdist_wheel docs including the --py-limited-api option:
+https://bitbucket.org/HexDecimal/wheel-fork/src/b25d28cc111f8d354e1b8d949a54ffb3c262dfce/docs/index.rst?fileviewer=file-view-default
+
+PEP 384: https://www.python.org/dev/peps/pep-0384/
+
+
+* Can I get rid of wx.siplib and just add dependency on SIP from pypi? Yes, of
+  course, but it would make using interim builds more difficult.
+
+* Can I eliminate all uses of the restricted APIs (at least the buffer related
+  things so far) and use equivalent APIs from siplib itself?  If not then I'll
+  probably have to separate out the incompatible stuff into my own API module
+  like siplib (and could probably include siplib there if desired...)
+
+* How to set the tags in the wheel filename? If bdist_wheel doesn't do it
+  already then it looks like we can override a get_tag(self) method in the
+  customization of the bdist_wheel class we already have...
+
+  For example, from py2exe's setup.py:
+
+    class my_bdist_wheel(bdist_wheel.bdist_wheel):
+        """We change the bdist_wheel command so that it creates a
+        wheel-file compatible with CPython, 3.4, 3.5, and 3.6 only
+        by setting the impl_tag to 'cp33.cp34.cp35.cp36'
+        """
+        def get_tag(self):
+            impl_tag, abi_tag, plat_tag = super().get_tag()
+            return "cp33.cp34.cp35.cp36", abi_tag, plat_tag
+
+
+* The bdist_wheel docs say that --py-limited-api is ignored on Windows???
+
+* The extension modules will need to have a different name, I think. Currently
+  waf is controlling that. Do current versions of waf support py-limited-api
+  or will something new need to be hacked into it?
+
+* Check into what Extension(py_limited_api=True, ...) does in recent versions
+  of setuptools.
+
+
+
+
+Issues discovered so far
+------------------------
+
+ * We WILL need a separate Python version-specific API module, siplib can give
+   us lots of help, but doesn't cover everything
+
+ * all the PyFoo_GET_ITEM type of "faster" macros will need to be changed to
+   their function call equivalents, meaning that our optimizations for those
+   things will be undone.
+
+ * At least most of the new buffer APIs are not available in the limited API
+
+ * It appears that PyDateTime APIs are also not available
+
+ * We access elements of PyMethod and such in a few places, in the limited API
+   those are opaque types.
+
+ * Ditto for sipSimpleWrapper
+
+ * There are a few things currently in wxpy_api like wxPyNoAppError that are
+   expected to end up in the _core module.