diff --git a/dill/__init__.py b/dill/__init__.py index bd3289f0..5f8a9940 100644 --- a/dill/__init__.py +++ b/dill/__init__.py @@ -26,7 +26,8 @@ from .dill import dump, dumps, load, loads, dump_session, load_session, \ Pickler, Unpickler, register, copy, pickle, pickles, HIGHEST_PROTOCOL, \ DEFAULT_PROTOCOL, PicklingError, UnpicklingError, \ - _revert_extension as revert_extension, _extend as extend + _revert_extension as revert_extension, _extend as extend, FMODE_NEWHANDLE, \ + FMODE_PRESERVEDATA, FMODE_PICKLECONTENTS from . import source, temp, detect # make sure "trace" is turned off diff --git a/dill/dill.py b/dill/dill.py index 9600ad81..4cd528cc 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -14,10 +14,11 @@ Test against "all" python types (Std. Lib. CH 1-15 @ 2.7) by mmckerns. Test against CH16+ Std. Lib. ... TBD. """ -__all__ = ['dump','dumps','load','loads','dump_session','load_session',\ - 'Pickler','Unpickler','register','copy','pickle','pickles',\ - 'HIGHEST_PROTOCOL','DEFAULT_PROTOCOL',\ - 'PicklingError','UnpicklingError'] +__all__ = ['dump','dumps','load','loads','dump_session','load_session', + 'Pickler','Unpickler','register','copy','pickle','pickles', + 'HIGHEST_PROTOCOL','DEFAULT_PROTOCOL', + 'PicklingError','UnpicklingError','FMODE_NEWHANDLE', + 'FMODE_PRESERVEDATA','FMODE_PICKLECONTENTS'] import logging log = logging.getLogger("dill") @@ -134,18 +135,31 @@ def ndarrayinstance(obj): return False except NameError: ExitType = None singletontypes = [] +### File modes +# pickles the file handle, preserving mode, with position of the unpickled +# object as for a new file handle. +FMODE_NEWHANDLE = 0 +# preserves existing data or create file if is does not exist, with +# position = min(pickled position, EOF), and mode which preserves behaviour +FMODE_PRESERVEDATA = 1 +# pickles the file handle, preserving mode and position, as well as the file +# contents +FMODE_PICKLECONTENTS = 2 + ### Shorthands (modified from python2.5/lib/pickle.py) def copy(obj, *args, **kwds): """use pickling to 'copy' an object""" return loads(dumps(obj, *args, **kwds)) -def dump(obj, file, protocol=None, byref=False): +def dump(obj, file, protocol=None, byref=False, file_mode=FMODE_NEWHANDLE, safeio=False): """pickle an object to a file""" if protocol is None: protocol = DEFAULT_PROTOCOL pik = Pickler(file, protocol) pik._main_module = _main_module _byref = pik._byref pik._byref = bool(byref) + pik._safeio = safeio + pik._file_mode = file_mode # hack to catch subclassed numpy array instances if NumpyArrayType and ndarrayinstance(obj): @register(type(obj)) @@ -160,10 +174,10 @@ def save_numpy_array(pickler, obj): pik._byref = _byref return -def dumps(obj, protocol=None, byref=False): +def dumps(obj, protocol=None, byref=False, file_mode=FMODE_NEWHANDLE, safeio=False): """pickle an object to a string""" file = StringIO() - dump(obj, file, protocol, byref) + dump(obj, file, protocol, byref, file_mode, safeio) return file.getvalue() def load(file): @@ -232,6 +246,8 @@ class Pickler(StockPickler): _main_module = None _session = False _byref = False + _safe_file = False + _file_mode = FMODE_NEWHANDLE pass def __init__(self, *args, **kwargs): @@ -356,27 +372,88 @@ def _create_lock(locked, *args): raise UnpicklingError("Cannot acquire lock") return lock -def _create_filehandle(name, mode, position, closed, open=open): # buffering=0 +def _create_filehandle(name, mode, position, closed, open, safeio, file_mode, fdata): # buffering=0 # only pickles the handle, not the file contents... good? or StringIO(data)? # (for file contents see: http://effbot.org/librarybook/copy-reg.htm) # NOTE: handle special cases first (are there more special cases?) names = {'':sys.__stdin__, '':sys.__stdout__, '':sys.__stderr__} #XXX: better fileno=(0,1,2) ? - if name in list(names.keys()): f = names[name] #XXX: safer "f=sys.stdin" - elif name == '': import os; f = os.tmpfile() - elif name == '': import tempfile; f = tempfile.TemporaryFile(mode) + if name in list(names.keys()): + f = names[name] #XXX: safer "f=sys.stdin" + elif name == '': + f = os.tmpfile() + elif name == '': + import tempfile + f = tempfile.TemporaryFile(mode) else: - try: # try to open the file by name # NOTE: has different fileno - f = open(name, mode)#FIXME: missing: *buffering*, encoding,softspace - except IOError: + # treat x mode as w mode + if "x" in mode and sys.hexversion < 0x03030000: + raise IOError("invalid mode 'x'") + + if not os.path.exists(name): + if safeio: + raise IOError("File '%s' does not exist" % name) + elif "r" in mode and file_mode != FMODE_PICKLECONTENTS: + name = os.devnull + current_size = 0 + else: + current_size = os.path.getsize(name) + + if position > current_size: + if safeio: + raise IOError("File '%s' is too short" % name) + elif file_mode == FMODE_PRESERVEDATA: + position = current_size + # try to open the file by name + # NOTE: has different fileno + try: + #FIXME: missing: *buffering*, encoding, softspace + if file_mode == FMODE_PICKLECONTENTS: + f = open(name, mode if "w" in mode else "w") + f.write(fdata) + if "w" not in mode: + f.close() + f = open(name, mode) + elif file_mode == FMODE_PRESERVEDATA \ + and ("w" in mode or "x" in mode): + # stop truncation when opening + flags = os.O_CREAT + if "+" in mode: + flags |= os.O_RDWR + else: + flags |= os.O_WRONLY + f = os.fdopen(os.open(name, flags), mode) + # set name to the correct value + if PY3: + r = getattr(f, "buffer", f) + r = getattr(r, "raw", r) + r.name = name + else: + class FILE(ctypes.Structure): + _fields_ = [("refcount", ctypes.c_long), + ("type_obj", ctypes.py_object), + ("file_pointer", ctypes.c_voidp), + ("name", ctypes.py_object)] + + class PyObject(ctypes.Structure): + _fields_ = [ + ("ob_refcnt", ctypes.c_int), + ("ob_type", ctypes.py_object) + ] + if not HAS_CTYPES: + raise RuntimeError("Need ctypes to set file name") + ctypes.cast(id(f), ctypes.POINTER(FILE)).contents.name = name + ctypes.cast(id(name), ctypes.POINTER(PyObject)).contents.ob_refcnt += 1 + assert f.name == name + else: + f = open(name, mode) + except IOError: err = sys.exc_info()[1] - try: # failing, then use /dev/null #XXX: better to just fail here? - import os; f = open(os.devnull, mode) - except IOError: - raise UnpicklingError(err) - #XXX: python default is closed '' file/mode - if closed: f.close() - elif position >= 0: f.seek(position) + raise UnpicklingError(err) + if closed: + f.close() + elif position >= 0 and file_mode != FMODE_NEWHANDLE: + f.seek(position) return f def _create_stringi(value, position, closed): @@ -583,15 +660,8 @@ def save_attrgetter(pickler, obj): pickler.save_reduce(type(obj), tuple(attrs), obj=obj) return -# __getstate__ explicitly added to raise TypeError when pickling: -# http://www.gossamer-threads.com/lists/python/bugs/871199 -@register(FileType) #XXX: in 3.x has buffer=0, needs different _create? -@register(BufferedRandomType) -@register(BufferedReaderType) -@register(BufferedWriterType) -@register(TextWrapperType) -def save_file(pickler, obj): - log.info("Fi: %s" % obj) +def _save_file(pickler, obj, open_): + obj.flush() if obj.closed: position = None else: @@ -599,24 +669,35 @@ def save_file(pickler, obj): position = -1 else: position = obj.tell() - pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, \ - obj.closed), obj=obj) + if pickler._file_mode == FMODE_PICKLECONTENTS: + f = open_(obj.name, "r") + fdata = f.read() + f.close() + else: + fdata = "" + pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, + obj.closed, open_, pickler._safeio, + pickler._file_mode, fdata), obj=obj) return -if PyTextWrapperType: #XXX: are stdout, stderr or stdin ever _pyio files? + +@register(FileType) #XXX: in 3.x has buffer=0, needs different _create? +@register(BufferedRandomType) +@register(BufferedReaderType) +@register(BufferedWriterType) +@register(TextWrapperType) +def save_file(pickler, obj): + log.info("Fi: %s" % obj) + return _save_file(pickler, obj, open) + +if PyTextWrapperType: @register(PyBufferedRandomType) @register(PyBufferedReaderType) @register(PyBufferedWriterType) @register(PyTextWrapperType) def save_file(pickler, obj): log.info("Fi: %s" % obj) - if obj.closed: - position = None - else: - position = obj.tell() - pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, \ - obj.closed, _open), obj=obj) - return + return _save_file(pickler, obj, _open) # The following two functions are based on 'saveCStringIoInput' # and 'saveCStringIoOutput' from spickle diff --git a/tests/test_file.py b/tests/test_file.py index 8571db4e..e0b5ad0a 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -1,4 +1,4 @@ -#usr/bin/env python +#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2008-2014 California Institute of Technology. @@ -38,14 +38,14 @@ def throws(op, args, exc): return False -def test(safefmode=False, kwargs={}): +def test(safeio, file_mode): # file exists, with same contents # read write_randomness() f = open(fname, "r") - _f = dill.loads(dill.dumps(f, **kwargs)) + _f = dill.loads(dill.dumps(f, safeio=safeio, file_mode=file_mode)) assert _f.mode == f.mode assert _f.tell() == f.tell() assert _f.read() == f.read() @@ -56,36 +56,32 @@ def test(safefmode=False, kwargs={}): f = open(fname, "w") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() f2 = dill.loads(f_dumped) f2mode = f2.mode f2tell = f2.tell() + f2name = f2.name f2.write(" world!") f2.close() - # 1) preserve mode and position - assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - assert f2mode == fmode - assert f2tell == ftell - # 2) treat as if new filehandle, will truncate file - # assert open(fname).read() == " world!" - # assert f2mode == fmode - # assert f2tell == 0 - # 3) prefer data over filehandle state - # assert open(fname).read() == "hello world!" - # assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? - # assert f2tell == ftell - # 4) use "r" to read data, then use "w" to write new file - # assert open(fname).read() == "hello world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2mode == fmode - # assert f2tell == ftell + if file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == " world!" + assert f2mode == fmode + assert f2tell == 0 + elif file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == "hello world!" + assert f2mode == fmode + assert f2tell == ftell + assert f2name == fname + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2mode == fmode + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") # append @@ -93,7 +89,7 @@ def test(safefmode=False, kwargs={}): f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -104,18 +100,17 @@ def test(safefmode=False, kwargs={}): f2.close() assert f2mode == fmode - # 1) preserve mode and position # also 3) - assert open(fname).read() == "hello world!" - assert f2tell == ftell - # 2) treat as if new filehandle, will seek(EOF) - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell - # 4) use "r" to read data, then use "a" to write new file - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") # file exists, with different contents (smaller size) # read @@ -124,46 +119,36 @@ def test(safefmode=False, kwargs={}): f = open(fname, "r") fstr = f.read() - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() _flen = 150 _fstr = write_randomness(number=_flen) - if safefmode: # throw error if ftell > EOF + if safeio: # throw error if ftell > EOF assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) assert f2.mode == fmode - # 1) preserve mode and position #XXX: ? - assert f2.tell() == ftell # 200 - assert f2.read() == "" - f2.seek(0) - assert f2.read() == _fstr - assert f2.tell() == _flen # 150 - # 3) prefer data over filehandle state - # assert f2.tell() == ftell # 200 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 150 - # 4) preserve mode and position, seek(EOF) if ftell > EOF - # assert f2.tell() == _flen # 150 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 150 - # 2) treat as if new filehandle, will seek(0) - # assert f2.tell() == 0 - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 150 - # 5) pickle data along with filehandle - # assert f2.tell() == ftell # 200 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == fstr - # assert f2.tell() == ftell # 200 + if file_mode == dill.FMODE_PRESERVEDATA: + assert f2.tell() == _flen + assert f2.read() == "" + f2.seek(0) + assert f2.read() == _fstr + assert f2.tell() == _flen # 150 + elif file_mode == dill.FMODE_NEWHANDLE: + assert f2.tell() == 0 + assert f2.read() == _fstr + assert f2.tell() == _flen # 150 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert f2.tell() == ftell # 200 + assert f2.read() == "" + f2.seek(0) + assert f2.read() == fstr + assert f2.tell() == ftell # 200 + else: + raise RuntimeError("Uncovered file mode!") f2.close() # write @@ -172,7 +157,7 @@ def test(safefmode=False, kwargs={}): f = open(fname, "w") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -183,7 +168,7 @@ def test(safefmode=False, kwargs={}): _ftell = f.tell() f.close() - if safefmode: # throw error if ftell > EOF + if safeio: # throw error if ftell > EOF assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -191,39 +176,29 @@ def test(safefmode=False, kwargs={}): f2tell = f2.tell() f2.write(" world!") f2.close() - # 1) preserve mode and position - assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - assert f2mode == fmode - assert f2tell == ftell - # 3) prefer data over filehandle state - # assert open(fname).read() == "h\x00\x00\x00\x00 world!" - # assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? - # assert f2tell == ftell - # 2) treat as if new filehandle, will truncate file - # assert open(fname).read() == " world!" - # assert f2mode == fmode - # assert f2tell == 0 - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 4a) use "r" to read data, then use "w" to write new file - # assert open(fname).read() == "h\x00\x00\x00\x00 world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 4b) preserve mode and position, seek(EOF) if ftell > EOF - # assert open(fname).read() == "h world!" - # assert f2mode == fmode - # assert f2tell == _ftell + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == "h world!" + assert f2mode == fmode + assert f2tell == _ftell + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == " world!" + assert f2mode == fmode + assert f2tell == 0 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2mode == fmode + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") f2.close() # append - write_randomness() + trunc_file() f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -234,7 +209,7 @@ def test(safefmode=False, kwargs={}): _ftell = f.tell() f.close() - if safefmode: # throw error if ftell > EOF + if safeio: # throw error if ftell > EOF assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -243,22 +218,18 @@ def test(safefmode=False, kwargs={}): f2.write(" world!") f2.close() assert f2mode == fmode - # 1) preserve mode and position # also 3) - # position of writes cannot be changed on some OSs - assert open(fname).read() == "h world!" - assert f2tell == ftell - # 2) treat as if new filehandle, will seek(EOF) - # assert open(fname).read() == "h world!" - # assert f2tell == _ftell - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell - # 4a) use "r" to read data, then use "a" to write new file - # assert open(fname).read() == "h world!" - # assert f2tell == ftell - # 4b) preserve mode and position, seek(EOF) if ftell > EOF - # assert open(fname).read() == "h world!" - # assert f2tell == _ftell + if file_mode == dill.FMODE_PRESERVEDATA: + # position of writes cannot be changed on some OSs + assert open(fname).read() == "h world!" + assert f2tell == _ftell + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == "h world!" + assert f2tell == _ftell + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") f2.close() # file does not exist @@ -268,46 +239,37 @@ def test(safefmode=False, kwargs={}): f = open(fname, "r") fstr = f.read() - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() os.remove(fname) - if safefmode: # throw error if file DNE + if safeio: # throw error if file DNE assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) assert f2.mode == fmode - # 1) preserve mode and position #XXX: ? - assert f2.tell() == ftell # 200 - assert f2.read() == "" - f2.seek(0) - assert f2.read() == "" - assert f2.tell() == 0 - # 3) prefer data over filehandle state - # assert f2.tell() == ftell # 200 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == "" - # assert f2.tell() == 0 - # 5) pickle data along with filehandle - # assert f2.tell() == ftell # 200 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == fstr - # assert f2.tell() == ftell # 200 - # 2) treat as if new filehandle, will seek(0) - # assert f2.tell() == 0 - # assert f2.read() == "" - # assert f2.tell() == 0 - # 4) preserve mode and position, seek(EOF) if ftell > EOF - # assert f2.tell() == 0 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == "" - # assert f2.tell() == 0 + if file_mode == dill.FMODE_PRESERVEDATA: + # FIXME: this fails on systems where f2.tell() always returns 0 + # assert f2.tell() == ftell # 200 + assert f2.read() == "" + f2.seek(0) + assert f2.read() == "" + assert f2.tell() == 0 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert f2.tell() == ftell # 200 + assert f2.read() == "" + f2.seek(0) + assert f2.read() == fstr + assert f2.tell() == ftell # 200 + elif file_mode == dill.FMODE_NEWHANDLE: + assert f2.tell() == 0 + assert f2.read() == "" + assert f2.tell() == 0 + else: + raise RuntimeError("Uncovered file mode!") f2.close() # write @@ -316,14 +278,14 @@ def test(safefmode=False, kwargs={}): f = open(fname, "w+") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) ftell = f.tell() fmode = f.mode f.close() os.remove(fname) - if safefmode: # throw error if file DNE + if safeio: # throw error if file DNE assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -331,30 +293,20 @@ def test(safefmode=False, kwargs={}): f2tell = f2.tell() f2.write(" world!") f2.close() - # 1) preserve mode and position - assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - assert f2mode == fmode - assert f2tell == ftell - # 3) prefer data over filehandle state - # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - # assert f2mode == 'w+' - # assert f2tell == ftell - # 2) treat as if new filehandle, will truncate file - # assert open(fname).read() == " world!" - # assert f2mode == fmode - # assert f2tell == 0 - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 4a) use "r" to read data, then use "w" to write new file - # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 4b) preserve mode and position, seek(EOF) if ftell > EOF - # assert open(fname).read() == " world!" - # assert f2mode == fmode - # assert f2tell == 0 + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == " world!" + assert f2mode == 'w+' + assert f2tell == 0 + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == " world!" + assert f2mode == fmode + assert f2tell == 0 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2mode == fmode + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") # append @@ -362,14 +314,14 @@ def test(safefmode=False, kwargs={}): f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) ftell = f.tell() fmode = f.mode f.close() os.remove(fname) - if safefmode: # throw error if file DNE + if safeio: # throw error if file DNE assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -378,21 +330,17 @@ def test(safefmode=False, kwargs={}): f2.write(" world!") f2.close() assert f2mode == fmode - # 1) preserve mode and position #XXX: also 3) - assert open(fname).read() == " world!" - assert f2tell == ftell - # 2) treat as if new filehandle, will seek(EOF) - # assert open(fname).read() == " world!" - # assert f2tell == 0 - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell - # 4a) use "r" to read data, then use "a" to write new file - # assert open(fname).read() == " world!" - # assert f2tell == ftell - # 4b) preserve mode and position, seek(EOF) if ftell > EOF - # assert open(fname).read() == " world!" - # assert f2tell == 0 + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == " world!" + assert f2tell == 0 + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == " world!" + assert f2tell == 0 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") # file exists, with different contents (larger size) # read @@ -401,89 +349,72 @@ def test(safefmode=False, kwargs={}): f = open(fname, "r") fstr = f.read() - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() _flen = 250 _fstr = write_randomness(number=_flen) - #XXX: no safefmode: no way to be 'safe'? + # XXX: no safe_file: no way to be 'safe'? f2 = dill.loads(f_dumped) assert f2.mode == fmode - # 1) preserve mode and position #XXX: ? - assert f2.tell() == ftell # 200 - assert f2.read() == _fstr[ftell:] - f2.seek(0) - assert f2.read() == _fstr - assert f2.tell() == _flen # 250 - # 3) prefer data over filehandle state - # assert f2.tell() == ftell # 200 - # assert f2.read() == _fstr[ftell:] - # f2.seek(0) - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 250 - # 4) preserve mode and position, seek(EOF) if ftell > EOF - # assert f2.tell() == ftell # 200 - # assert f2.read() == _fstr[ftell:] - # f2.seek(0) - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 250 - # 2) treat as if new filehandle, will seek(0) - # assert f2.tell() == 0 - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 250 - # 5) pickle data along with filehandle - # assert f2.tell() == ftell # 200 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == fstr - # assert f2.tell() == ftell # 200 - f2.close() #XXX: other alternatives? + if file_mode == dill.FMODE_PRESERVEDATA: + assert f2.tell() == ftell # 200 + assert f2.read() == _fstr[ftell:] + f2.seek(0) + assert f2.read() == _fstr + assert f2.tell() == _flen # 250 + elif file_mode == dill.FMODE_NEWHANDLE: + assert f2.tell() == 0 + assert f2.read() == _fstr + assert f2.tell() == _flen # 250 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert f2.tell() == ftell # 200 + assert f2.read() == "" + f2.seek(0) + assert f2.read() == fstr + assert f2.tell() == ftell # 200 + else: + raise RuntimeError("Uncovered file mode!") + f2.close() # XXX: other alternatives? # write f = open(fname, "w") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() -# f.close() + fstr = open(fname).read() -# f = open(fname, "a") f.write(" and goodbye!") _ftell = f.tell() f.close() - #XXX: no safefmode: no way to be 'safe'? + # XXX: no safe_file: no way to be 'safe'? f2 = dill.loads(f_dumped) f2mode = f2.mode f2tell = f2.tell() f2.write(" world!") f2.close() - # 1) preserve mode and position - assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - assert f2mode == fmode - assert f2tell == ftell - # 3) prefer data over filehandle state - # assert open(fname).read() == "hello world!odbye!" - # assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? - # assert f2tell == ftell - # 2) treat as if new filehandle, will truncate file - # assert open(fname).read() == " world!" - # assert f2mode == fmode - # assert f2tell == 0 - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 4) use "r" to read data, then use "w" to write new file - # assert open(fname).read() == "hello world!odbye!" - # assert f2mode == fmode - # assert f2tell == ftell + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == "hello world!odbye!" + assert f2mode == fmode + assert f2tell == ftell + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == " world!" + assert f2mode == fmode + assert f2tell == 0 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2mode == fmode + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") f2.close() # append @@ -492,7 +423,7 @@ def test(safefmode=False, kwargs={}): f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() fstr = open(fname).read() @@ -501,7 +432,7 @@ def test(safefmode=False, kwargs={}): _ftell = f.tell() f.close() - #XXX: no safefmode: no way to be 'safe'? + # XXX: no safe_file: no way to be 'safe'? f2 = dill.loads(f_dumped) f2mode = f2.mode @@ -509,23 +440,27 @@ def test(safefmode=False, kwargs={}): f2.write(" world!") f2.close() assert f2mode == fmode - # 1) preserve mode and position # also 3) - assert open(fname).read() == "hello and goodbye! world!" - assert f2tell == ftell - # 2) treat as if new filehandle, will seek(EOF) - # assert open(fname).read() == "hello and goodbye! world!" - # assert f2tell == _ftell - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell - # 4) use "r" to read data, then use "a" to write new file - # assert open(fname).read() == "hello and goodbye! world!" - # assert f2tell == ftell + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == "hello and goodbye! world!" + assert f2tell == ftell + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == "hello and goodbye! world!" + assert f2tell == _ftell + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") f2.close() -test() -# TODO: switch this on when #57 is closed -# test(True, {"safe_file": True}) +test(safeio=False, file_mode=dill.FMODE_NEWHANDLE) +test(safeio=False, file_mode=dill.FMODE_PRESERVEDATA) +test(safeio=False, file_mode=dill.FMODE_PICKLECONTENTS) + +test(safeio=True, file_mode=dill.FMODE_NEWHANDLE) +test(safeio=True, file_mode=dill.FMODE_PRESERVEDATA) +test(safeio=True, file_mode=dill.FMODE_PICKLECONTENTS) + if os.path.exists(fname): os.remove(fname)