Issue #127: use C-Python pickling for 'iterator' and 'callable_iterator'

Disable the Stackless specific code for pickling 'iterator' and 'callable_iterator' objects. C-Python 3.3 already pickles them.
Add tests to ensure, that Stackless can still unpickle old pickles.

https://bitbucket.org/stackless-dev/stackless/issues/127

Author: Anselm Kruis

Stackless/changelog.txt

@@ -10,9 +10,11 @@ What's New in Stackless 3.X.X?
 *Release date: 20XX-XX-XX*
 
 - https://bitbucket.org/stackless-dev/stackless/issues/127
-  Fix pickling of 'callable-iterator' and 'method-wrapper' objects.
-  Fix copy.copy() for 'callable-iterator', 'method', 'dict_keys', 'dict_values'
-  and 'dict_items' objects.
+  Disable the Stackless specific code for pickling 'iterator' and
+  'callable_iterator' objects. C-Python 3.3 already pickles them.
+  Fix pickling of 'method-wrapper' objects.
+  Fix copy.copy() for 'method', 'dict_keys', 'dict_values' and 'dict_items'
+  objects.
 
 - https://bitbucket.org/stackless-dev/stackless/issues/126
   Load the module stackless early in all C-Python tests. This ensures a defined

Stackless/pickling/prickelpit.c

@@ -201,7 +201,8 @@ static struct PyMethodDef _new_methoddef[] = {
     {0}
 };
 
-static int init_type(PyTypeObject *t, int (*initchain)(void))
+static int 
+init_type_ex(PyTypeObject *t, int (*initchain)(void), int register_reduce)
 {
     PyMethodDescrObject *reduce;
     PyWrapperDescrObject *init;
@@ -230,15 +231,18 @@ static int init_type(PyTypeObject *t, int (*initchain)(void))
     if (func == NULL || PyDict_SetItemString(t->tp_dict, "__new__", func))
         return -1;
     /* register with copy_reg */
-    if (pickle_reg != NULL &&
-        (retval = PyObject_CallFunction(pickle_reg, "OO",
-                                        t->tp_base, reduce)) == NULL)
-        ret = -1;
+    if (register_reduce) {
+        if (pickle_reg != NULL &&
+            (retval = PyObject_CallFunction(pickle_reg, "OO",
+                                            t->tp_base, reduce)) == NULL)
+            ret = -1;
+    }
     Py_XDECREF(retval);
     if (ret == 0 && initchain != NULL)
         ret = initchain();
     return ret;
 }
+#define init_type(t_, initchain_) init_type_ex((t_), (initchain_), 1)
 
 /* root of init function chain */
 
@@ -1427,8 +1431,8 @@ MAKE_WRAPPERTYPE(PyCallIter_Type, calliter, "callable_iterator",
 
 static int init_itertype(void)
 {
-    return init_type(&wrap_PySeqIter_Type, NULL)
-        || init_type(&wrap_PyCallIter_Type, initchain);
+    return init_type_ex(&wrap_PySeqIter_Type, NULL, 0)
+        || init_type_ex(&wrap_PyCallIter_Type, initchain, 0);
 }
 #undef initchain
 #define initchain init_itertype

Stackless/unittests/test_pickle.py

@@ -716,6 +716,77 @@ def test_dict_items(self):
         obj = d.items()
         self._test(obj)
 
+
+class SimpleSequence:
+    def __getitem__(self, n):
+        if n < 3:
+            return n
+        raise IndexError
+
+
+class TestOldStackless_WrapFactories(StacklessPickleTestCase):
+    def test_iterator(self):
+        # Stackless prior to version 3.3.7 used to register its own __reduce__
+        # method for 'iterator' with copy_reg. This method returned the function
+        # stackless._wrap.iterator as iterator factory.
+        #
+        # This test case ensures that stackless can still unpickle old pickles:
+
+        # an old pickle, protocol 0 of iter(SimpleSequence())
+        p = b'c_stackless._wrap\niterator\n(ccopy_reg\n_reconstructor\n(ctest_pickle\nSimpleSequence\nc__builtin__\nobject\nNtRL0L\ntR(tb.'
+
+        # Output of
+        # from pickletools import dis; dis(p)
+        #    0: c    GLOBAL     '_stackless._wrap iterator'
+        #   27: (    MARK
+        #   28: c        GLOBAL     'copy_reg _reconstructor'
+        #   53: (        MARK
+        #   54: c            GLOBAL     'test_pickle SimpleSequence'
+        #   82: c            GLOBAL     '__builtin__ object'
+        #  102: N            NONE
+        #  103: t            TUPLE      (MARK at 53)
+        #  104: R        REDUCE
+        #  105: L        LONG       0
+        #  109: t        TUPLE      (MARK at 27)
+        #  110: R    REDUCE
+        #  111: (    MARK
+        #  112: t        TUPLE      (MARK at 111)
+        #  113: b    BUILD
+        #  114: .    STOP
+        # highest protocol among opcodes = 0
+
+        x = self.loads(p)
+        self.assertIs(type(x), type(iter(SimpleSequence())))
+        self.assertListEqual(list(x), [0, 1, 2])
+
+    def test_callable_iterator(self):
+        # Stackless prior to version 3.3.7 used to register its own __reduce__
+        # method for 'callable_iterator' with copy_reg. This method returned the function
+        # stackless._wrap.callable_iterator as callable_iterator factory.
+        #
+        # This test case ensures that stackless can still unpickle old pickles:
+
+        # an old pickle, protocol 0 of iter(bool, False)
+        p = b'c_stackless._wrap\ncallable_iterator\n(c__builtin__\nbool\nI00\ntR.'
+
+        # Output of
+        # from pickletools import dis; dis(p)
+        #    0: c    GLOBAL     '_stackless._wrap callable_iterator'
+        #   36: (    MARK
+        #   37: c        GLOBAL     '__builtin__ bool'
+        #   55: I        INT        False
+        #   59: t        TUPLE      (MARK at 36)
+        #   60: R    REDUCE
+        #   61: .    STOP
+        # highest protocol among opcodes = 0
+
+        x = self.loads(p)
+        # Use assertIsInstance, because x may be a subclass of i.
+        # See bug https://bitbucket.org/stackless-dev/stackless/issues/127
+        self.assertIsInstance(x, type(iter(bool, False)))
+        self.assertListEqual(list(x), [])
+
+
 if __name__ == '__main__':
     if not sys.argv[1:]:
         sys.argv.append('-v')