renamed FMODEs; rename file_mode and safeio to fmode and strictio;

made dill.diff 2.5-compatible and disabled; move IOError to ValueError/FNFError
move ctypes RuntimeError to ImportError; test_file.throws takes an instance


git-svn-id: svn+ssh://svn.mystic.cacr.caltech.edu/pathos/dill@719 8bfda07e-5b16-0410-ab1d-fd04ec2748df

dill/diff.py → dill/__diff.py

@@ -28,9 +28,9 @@
 memo = {}
 id_to_obj = {}
 # types that cannot have changing attributes
-builtins_types = {str, list, dict, set, frozenset, int}
-dont_memo = {id(i) for i in (memo, sys.modules, sys.path_importer_cache,
-             os.environ, id_to_obj)}
+builtins_types = set((str, list, dict, set, frozenset, int))
+dont_memo = set(id(i) for i in (memo, sys.modules, sys.path_importer_cache,
+             os.environ, id_to_obj))
 
 
 def get_attrs(obj):
@@ -46,7 +46,7 @@ def get_attrs(obj):
         return
 
 
-def get_seq(obj, cashe={str: False, frozenset: False, list: True, set: True,
+def get_seq(obj, cache={str: False, frozenset: False, list: True, set: True,
                         dict: True, tuple: True, type: False,
                         types.ModuleType: False, types.FunctionType: False,
                         types.BuiltinFunctionType: False}):
@@ -55,8 +55,8 @@ def get_seq(obj, cashe={str: False, frozenset: False, list: True, set: True,
     """
     o_type = type(obj)
     hsattr = hasattr
-    if o_type in cashe:
-        if cashe[o_type]:
+    if o_type in cache:
+        if cache[o_type]:
             if hsattr(obj, "copy"):
                 return obj.copy()
             return obj
@@ -68,12 +68,12 @@ def get_seq(obj, cashe={str: False, frozenset: False, list: True, set: True,
     elif hsattr(obj, "__contains__") and hsattr(obj, "__iter__") \
        and hsattr(obj, "__len__") and hsattr(o_type, "__contains__") \
        and hsattr(o_type, "__iter__") and hsattr(o_type, "__len__"):
-        cashe[o_type] = True
+        cache[o_type] = True
         if hsattr(obj, "copy"):
             return obj.copy()
         return obj
     else:
-        cashe[o_type] = False
+        cache[o_type] = False
         return None
 
 
@@ -91,13 +91,13 @@ def memorise(obj, force=False):
     if g is None:
         attrs_id = None
     else:
-        attrs_id = {key: id_(value) for key, value in g.items()}
+        attrs_id = dict((key,id_(value)) for key, value in g.items())
 
     s = get_seq(obj)
     if s is None:
         seq_id = None
     elif hasattr(s, "items"):
-        seq_id = {id_(key): id_(value) for key, value in s.items()}
+        seq_id = dict((id_(key),id_(value)) for key, value in s.items())
     else:
         seq_id = [id_(i) for i in s]
 
@@ -168,7 +168,7 @@ def whats_changed(obj, seen=None, simple=False, first=True):
     else:
         obj_attrs = memo[obj_id][0]
         obj_get = obj_attrs.get
-        changed = {key: None for key in obj_attrs if key not in attrs}
+        changed = dict((key,None) for key in obj_attrs if key not in attrs)
         for key, o in attrs.items():
             if id_(o) != obj_get(key, None) or chngd(o, seen, True, False):
                 changed[key] = o
@@ -199,19 +199,20 @@ def whats_changed(obj, seen=None, simple=False, first=True):
     return changed, seq_diff
 
 
-def has_changed(*args, **kwargs):
-    return whats_changed(*args, simple=True, **kwargs)
+def has_changed(*args, **kwds):
+    kwds['simple'] = True  # ignore simple if passed in
+    return whats_changed(*args, **kwds)
 
 __import__ = __import__
 
 
-def _imp(*args, **kwargs):
+def _imp(*args, **kwds):
     """
     Replaces the default __import__, to allow a module to be memorised
     before the user can change it
     """
     before = set(sys.modules.keys())
-    mod = __import__(*args, **kwargs)
+    mod = __import__(*args, **kwds)
     after = set(sys.modules.keys()).difference(before)
     for m in after:
         memorise(sys.modules[m])

dill/__init__.py

@@ -24,9 +24,9 @@
 """ + __license__
 
 from .dill import dump, dumps, load, loads, dump_session, load_session, \
-    Pickler, Unpickler, register, copy, pickle, pickles, use_diff, \
+    Pickler, Unpickler, register, copy, pickle, pickles, \
     HIGHEST_PROTOCOL, DEFAULT_PROTOCOL, PicklingError, UnpicklingError, \
-    FMODE_NEWHANDLE, FMODE_PRESERVEDATA, FMODE_PICKLECONTENTS
+    HANDLE_FMODE, CONTENTS_FMODE, FILE_FMODE
 from . import source, temp, detect
 
 # make sure "trace" is turned off

dill/dill.py

@@ -16,9 +16,8 @@
 """
 __all__ = ['dump','dumps','load','loads','dump_session','load_session',
            'Pickler','Unpickler','register','copy','pickle','pickles',
-           'use_diff', 'HIGHEST_PROTOCOL','DEFAULT_PROTOCOL',
-           'PicklingError','UnpicklingError','FMODE_NEWHANDLE',
-           'FMODE_PRESERVEDATA','FMODE_PICKLECONTENTS']
+           'HIGHEST_PROTOCOL','DEFAULT_PROTOCOL','PicklingError',
+           'UnpicklingError','HANDLE_FMODE','CONTENTS_FMODE','FILE_FMODE']
 
 import logging
 log = logging.getLogger("dill")
@@ -33,7 +32,7 @@ def _trace(boolean):
 import sys
 diff = None
 _use_diff = False
-PY3 = (hex(sys.hexversion) >= '0x30000f0')
+PY3 = (sys.hexversion >= 0x30000f0)
 if PY3: #XXX: get types from dill.objtypes ?
     import builtins as __builtin__
     from pickle import _Pickler as StockPickler, _Unpickler as StockUnpickler
@@ -71,8 +70,11 @@ def _trace(boolean):
 from functools import partial
 from operator import itemgetter, attrgetter
 # new in python2.5
-if hex(sys.hexversion) >= '0x20500f0':
+if sys.hexversion >= 0x20500f0:
     from types import MemberDescriptorType, GetSetDescriptorType
+# new in python3.3
+if sys.hexversion < 0x03030000:
+    FileNotFoundError = IOError
 try:
     import ctypes
     HAS_CTYPES = True
@@ -138,30 +140,30 @@ def ndarrayinstance(obj): return False
     singletontypes = []
 
 ### File modes
-# pickles the file handle, preserving mode, with position of the unpickled
-# object as for a new file handle.
-FMODE_NEWHANDLE = 0
-# preserves existing data or create file if is does not exist, with
-# position = min(pickled position, EOF), and mode which preserves behaviour
-FMODE_PRESERVEDATA = 1
-# pickles the file handle, preserving mode and position, as well as the file
-# contents
-FMODE_PICKLECONTENTS = 2
+# Pickles the file handle, preserving mode. The position of the unpickled
+# object is as for a new file handle.
+HANDLE_FMODE = 0
+# Pickles the file contents, creating a new file if on load the file does
+# not exist. The position = min(pickled position, EOF) and mode is chosen
+# as such that "best" preserves behavior of the original file.
+CONTENTS_FMODE = 1
+# Pickles the entire file (handle and contents), preserving mode and position.
+FILE_FMODE = 2
 
 ### Shorthands (modified from python2.5/lib/pickle.py)
 def copy(obj, *args, **kwds):
     """use pickling to 'copy' an object"""
     return loads(dumps(obj, *args, **kwds))
 
-def dump(obj, file, protocol=None, byref=False, file_mode=FMODE_NEWHANDLE, safeio=False):
+def dump(obj, file, protocol=None, byref=False, fmode=HANDLE_FMODE, strictio=False):
     """pickle an object to a file"""
     if protocol is None: protocol = DEFAULT_PROTOCOL
     pik = Pickler(file, protocol)
     pik._main_module = _main_module
     _byref = pik._byref
     pik._byref = bool(byref)
-    pik._safeio = safeio
-    pik._file_mode = file_mode
+    pik._strictio = strictio
+    pik._fmode = fmode
     # hack to catch subclassed numpy array instances
     if NumpyArrayType and ndarrayinstance(obj):
         @register(type(obj))
@@ -176,10 +178,10 @@ def save_numpy_array(pickler, obj):
     pik._byref = _byref
     return
 
-def dumps(obj, protocol=None, byref=False, file_mode=FMODE_NEWHANDLE, safeio=False):
+def dumps(obj, protocol=None, byref=False, fmode=HANDLE_FMODE, strictio=False):
     """pickle an object to a string"""
     file = StringIO()
-    dump(obj, file, protocol, byref, file_mode, safeio)
+    dump(obj, file, protocol, byref, fmode, strictio)
     return file.getvalue()
 
 def load(file):
@@ -249,13 +251,13 @@ class Pickler(StockPickler):
     _session = False
     _byref = False
     _safe_file = False
-    _file_mode = FMODE_NEWHANDLE
+    _fmode = HANDLE_FMODE
     pass
 
     def __init__(self, *args, **kwargs):
         StockPickler.__init__(self, *args, **kwargs)
         self._main_module = _main_module
-        self._diff_cashe = {}
+        self._diff_cache = {}
 
 class Unpickler(StockUnpickler):
     """python's Unpickler extended to interpreter sessions and more types"""
@@ -392,7 +394,7 @@ def _create_lock(locked, *args):
     return lock
 
 # thanks to matsjoyce for adding all the different file modes
-def _create_filehandle(name, mode, position, closed, open, safeio, file_mode, fdata): # buffering=0
+def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdata): # buffering=0
     # only pickles the handle, not the file contents... good? or StringIO(data)?
     # (for file contents see: http://effbot.org/librarybook/copy-reg.htm)
     # NOTE: handle special cases first (are there more special cases?)
@@ -408,33 +410,33 @@ def _create_filehandle(name, mode, position, closed, open, safeio, file_mode, fd
     else:
         # treat x mode as w mode
         if "x" in mode and sys.hexversion < 0x03030000:
-                raise IOError("invalid mode 'x'")
+            raise ValueError("invalid mode: '%s'" % mode)
 
         if not os.path.exists(name):
-            if safeio:
-                raise IOError("File '%s' does not exist" % name)
-            elif "r" in mode and file_mode != FMODE_PICKLECONTENTS:
+            if strictio:
+                raise FileNotFoundError("[Errno 2] No such file or directory: '%s'" % name)
+            elif "r" in mode and fmode != FILE_FMODE:
                 name = os.devnull
             current_size = 0
         else:
             current_size = os.path.getsize(name)
 
         if position > current_size:
-            if safeio:
-                raise IOError("File '%s' is too short" % name)
-            elif file_mode == FMODE_PRESERVEDATA:
+            if strictio:
+                raise ValueError("invalid buffer size")
+            elif fmode == CONTENTS_FMODE:
                 position = current_size
         # try to open the file by name
         # NOTE: has different fileno
         try:
             #FIXME: missing: *buffering*, encoding, softspace
-            if file_mode == FMODE_PICKLECONTENTS:
+            if fmode == FILE_FMODE:
                 f = open(name, mode if "w" in mode else "w")
                 f.write(fdata)
                 if "w" not in mode:
                     f.close()
                     f = open(name, mode)
-            elif file_mode == FMODE_PRESERVEDATA \
+            elif fmode == CONTENTS_FMODE \
                and ("w" in mode or "x" in mode):
                 # stop truncation when opening
                 flags = os.O_CREAT
@@ -461,7 +463,7 @@ class PyObject(ctypes.Structure):
                             ("ob_type", ctypes.py_object)
                             ]
                     if not HAS_CTYPES:
-                        raise RuntimeError("Need ctypes to set file name")
+                        raise ImportError("No module named 'ctypes'")
                     ctypes.cast(id(f), ctypes.POINTER(FILE)).contents.name = name
                     ctypes.cast(id(name), ctypes.POINTER(PyObject)).contents.ob_refcnt += 1
                 assert f.name == name
@@ -472,7 +474,7 @@ class PyObject(ctypes.Structure):
             raise UnpicklingError(err)
     if closed:
         f.close()
-    elif position >= 0 and file_mode != FMODE_NEWHANDLE:
+    elif position >= 0 and fmode != HANDLE_FMODE:
         f.seek(position)
     return f
 
@@ -689,15 +691,17 @@ def _save_file(pickler, obj, open_):
             position = -1
         else:
             position = obj.tell()
-    if pickler._file_mode == FMODE_PICKLECONTENTS:
+    if pickler._fmode == FILE_FMODE:
         f = open_(obj.name, "r")
         fdata = f.read()
         f.close()
     else:
         fdata = ""
+    strictio = pickler._strictio
+    fmode = pickler._fmode
     pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position,
-                                             obj.closed, open_, pickler._safeio,
-                                             pickler._file_mode, fdata), obj=obj)
+                                             obj.closed, open_, strictio,
+                                             fmode, fdata), obj=obj)
     return
 
 
@@ -779,7 +783,7 @@ def save_instancemethod0(pickler, obj):# example: cStringIO.StringI
                                          obj.im_class), obj=obj)
     return
 
-if hex(sys.hexversion) >= '0x20500f0':
+if sys.hexversion >= 0x20500f0:
     @register(MemberDescriptorType)
     @register(GetSetDescriptorType)
     @register(MethodDescriptorType)
@@ -904,10 +908,10 @@ def save_weakproxy(pickler, obj):
 
 @register(ModuleType)
 def save_module(pickler, obj):
-    if _use_diff:
+    if False: #_use_diff:
         if obj.__name__ != "dill":
             try:
-                changed = diff.whats_changed(obj, seen=pickler._diff_cashe)[0]
+                changed = diff.whats_changed(obj, seen=pickler._diff_cache)[0]
             except RuntimeError:  # not memorised module, probably part of dill
                 pass
             else:
@@ -1014,4 +1018,6 @@ def _extend():
         else: pass
     return
 
+del diff, _use_diff, use_diff
+
 # EOF

tests/test_diff.py

@@ -5,7 +5,7 @@
 # License: 3-clause BSD.  The full license text is available at:
 #  - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/dill/LICENSE
 
-from dill import diff
+from dill import __diff as diff
 
 
 class A:
@@ -50,16 +50,38 @@ class A:
 assert changed[0] == {}
 assert changed[1]
 
-import abc
-# make sure that the "_abc_invaldation_counter" does not cause the test to fail
-diff.memorise(abc.ABCMeta, force=True)
-assert not diff.has_changed(abc)
-abc.ABCMeta.zzz = 1
-assert diff.has_changed(abc)
-changed = diff.whats_changed(abc)
-assert list(changed[0].keys()) == ["ABCMeta"]
+try:
+    import abc
+    # make sure the "_abc_invaldation_counter" does not cause test to fail
+    diff.memorise(abc.ABCMeta, force=True)
+    assert not diff.has_changed(abc)
+    abc.ABCMeta.zzz = 1
+    assert diff.has_changed(abc)
+    changed = diff.whats_changed(abc)
+    assert list(changed[0].keys()) == ["ABCMeta"]
+    assert not changed[1]
+except ImportError:
+    pass
+
+'''
+import Queue
+diff.memorise(Queue, force=True)
+assert not diff.has_changed(Queue)
+Queue.Queue.zzz = 1
+assert diff.has_changed(Queue)
+changed = diff.whats_changed(Queue)
+assert list(changed[0].keys()) == ["Queue"]
 assert not changed[1]
 
+import math
+diff.memorise(math, force=True)
+assert not diff.has_changed(math)
+math.zzz = 1
+assert diff.has_changed(math)
+changed = diff.whats_changed(math)
+assert list(changed[0].keys()) == ["zzz"]
+assert not changed[1]
+'''
 
 a = A()
 b = A()