From 72882e949970d9ed81c7dac360998eecfce23feb Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Wed, 23 Jul 2014 18:42:27 +0100 Subject: [PATCH 01/11] Implement changes in #57: improve behaviour of dumped files --- dill/dill.py | 67 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index c158702b..28424d79 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -138,13 +138,14 @@ def copy(obj, *args, **kwds): """use pickling to 'copy' an object""" return loads(dumps(obj, *args, **kwds)) -def dump(obj, file, protocol=None, byref=False): +def dump(obj, file, protocol=None, byref=False, safe_file=False): """pickle an object to a file""" if protocol is None: protocol = DEFAULT_PROTOCOL pik = Pickler(file, protocol) pik._main_module = _main_module _byref = pik._byref pik._byref = bool(byref) + pik._safe_file = safe_file # hack to catch subclassed numpy array instances if NumpyArrayType and ndarrayinstance(obj): @register(type(obj)) @@ -159,10 +160,10 @@ def save_numpy_array(pickler, obj): pik._byref = _byref return -def dumps(obj, protocol=None, byref=False): +def dumps(obj, protocol=None, byref=False, safe_file=False): """pickle an object to a string""" file = StringIO() - dump(obj, file, protocol, byref) + dump(obj, file, protocol, byref, safe_file) return file.getvalue() def load(file): @@ -231,6 +232,7 @@ class Pickler(StockPickler): _main_module = None _session = False _byref = False + _safe_file = False pass def __init__(self, *args, **kwargs): @@ -355,27 +357,54 @@ def _create_lock(locked, *args): raise UnpicklingError("Cannot acquire lock") return lock -def _create_filehandle(name, mode, position, closed, open=open): # buffering=0 +def _create_filehandle(name, mode, position, closed, open=open, safe=False): # buffering=0 # only pickles the handle, not the file contents... good? or StringIO(data)? # (for file contents see: http://effbot.org/librarybook/copy-reg.htm) # NOTE: handle special cases first (are there more special cases?) names = {'':sys.__stdin__, '':sys.__stdout__, '':sys.__stderr__} #XXX: better fileno=(0,1,2) ? - if name in list(names.keys()): f = names[name] #XXX: safer "f=sys.stdin" - elif name == '': import os; f = os.tmpfile() - elif name == '': import tempfile; f = tempfile.TemporaryFile(mode) + if name in list(names.keys()): + f = names[name] #XXX: safer "f=sys.stdin" + elif name == '': + import os + f = os.tmpfile() + elif name == '': + import tempfile + f = tempfile.TemporaryFile(mode) else: - try: # try to open the file by name # NOTE: has different fileno - f = open(name, mode)#FIXME: missing: *buffering*, encoding,softspace - except IOError: + import os + # Mode translation + # Mode | Unpickled mode + # --------|--------------- + # r | r + # r+ | r+ + # w | r+ + # w+ | r+ + # a | a + # a+ | a+ + + if os.path.exists(name): + mode = mode.replace("w+", "r+") + mode = mode.replace("w", "r+") + elif safe: + raise IOError("File '%s' does not exist" % name) + elif "w" not in mode: + name = os.devnull + if safe: + if position > os.path.getsize(name): + raise IOError("File '%s' is too short" % name) + # try to open the file by name + # NOTE: has different fileno + try: + f = open(name, mode) #FIXME: missing: *buffering*, encoding, softspace + except IOError: err = sys.exc_info()[1] - try: # failing, then use /dev/null #XXX: better to just fail here? - import os; f = open(os.devnull, mode) - except IOError: - raise UnpicklingError(err) - #XXX: python default is closed '' file/mode - if closed: f.close() - elif position >= 0: f.seek(position) + raise UnpicklingError(err) + #XXX: python default is closed '' file/mode + if closed: + f.close() + elif position >= 0: + f.seek(position) return f def _create_stringi(value, position, closed): @@ -599,7 +628,7 @@ def save_file(pickler, obj): else: position = obj.tell() pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, \ - obj.closed), obj=obj) + obj.closed, open, pickler._safe_file), obj=obj) return if PyTextWrapperType: #XXX: are stdout, stderr or stdin ever _pyio files? @@ -614,7 +643,7 @@ def save_file(pickler, obj): else: position = obj.tell() pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, \ - obj.closed, _open), obj=obj) + obj.closed, _open, pickler._safe_file), obj=obj) return # The following two functions are based on 'saveCStringIoInput' From ac9ff01eb9fcb45f9167f74366e5a7b99aa1b2fc Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Mon, 4 Aug 2014 09:43:50 +0100 Subject: [PATCH 02/11] Fixes for test and due to test --- dill/dill.py | 6 +++++- tests/test_file.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index 28424d79..dcec4927 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -382,13 +382,17 @@ def _create_filehandle(name, mode, position, closed, open=open, safe=False): # b # w+ | r+ # a | a # a+ | a+ + # Note: If the file does not exist, the mode is not translated + + if mode == "x": + mode = "w" if os.path.exists(name): mode = mode.replace("w+", "r+") mode = mode.replace("w", "r+") elif safe: raise IOError("File '%s' does not exist" % name) - elif "w" not in mode: + elif "r" in mode: name = os.devnull if safe: if position > os.path.getsize(name): diff --git a/tests/test_file.py b/tests/test_file.py index 829e9a2f..fe86d3ba 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -335,7 +335,7 @@ def test(safefmode=False, kwargs={}): assert f2tell == ftell # 3) prefer data over filehandle state # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - # assert f2mode == 'r+' #FIXME: spec'd as 'r+' but is 'w+' + # assert f2mode == 'w+' # assert f2tell == ftell # 2) treat as if new filehandle, will truncate file # assert open(fname).read() == " world!" @@ -377,7 +377,7 @@ def test(safefmode=False, kwargs={}): f2.close() assert f2mode == fmode # 1) preserve mode and position #XXX: also 3) - assert open(fname).read() == " world!" # 3) FIXME: throws, should not? + assert open(fname).read() == " world!" # 3) assert f2tell == ftell # 2) treat as if new filehandle, will seek(EOF) # assert open(fname).read() == " world!" From 06150ac4100ab60dc25741faf19d07eef5aa86be Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Tue, 5 Aug 2014 21:19:14 +0100 Subject: [PATCH 03/11] Change file pickling to avoid \x00 chars --- dill/dill.py | 3 +- tests/test_file.py | 87 +++++++++++++++++++++++----------------------- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index dcec4927..6a3fe137 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -408,7 +408,8 @@ def _create_filehandle(name, mode, position, closed, open=open, safe=False): # b if closed: f.close() elif position >= 0: - f.seek(position) + eof = os.path.getsize(name) + f.seek(position if position < eof else eof) return f def _create_stringi(value, position, closed): diff --git a/tests/test_file.py b/tests/test_file.py index fe86d3ba..3fd03d14 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -65,17 +65,17 @@ def test(safefmode=False, kwargs={}): f2.close() # 1) preserve mode and position - assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - assert f2mode == fmode - assert f2tell == ftell + # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" + # assert f2mode == fmode + # assert f2tell == ftell # 2) treat as if new filehandle, will truncate file # assert open(fname).read() == " world!" # assert f2mode == fmode # assert f2tell == 0 # 3) prefer data over filehandle state - # assert open(fname).read() == "hello world!" - # assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? - # assert f2tell == ftell + assert open(fname).read() == "hello world!" + assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? + assert f2tell == ftell # 4) use "r" to read data, then use "w" to write new file # assert open(fname).read() == "hello world!" # assert f2mode == fmode @@ -135,17 +135,17 @@ def test(safefmode=False, kwargs={}): f2 = dill.loads(f_dumped) assert f2.mode == fmode # 1) preserve mode and position #XXX: ? - assert f2.tell() == ftell # 200 - assert f2.read() == "" - f2.seek(0) - assert f2.read() == _fstr - assert f2.tell() == _flen # 150 - # 3) prefer data over filehandle state # assert f2.tell() == ftell # 200 # assert f2.read() == "" # f2.seek(0) # assert f2.read() == _fstr # assert f2.tell() == _flen # 150 + # 3) prefer data over filehandle state + assert f2.tell() == _flen + assert f2.read() == "" + f2.seek(0) + assert f2.read() == _fstr + assert f2.tell() == _flen # 150 # 4) preserve mode and position, seek(EOF) if ftell > EOF # assert f2.tell() == _flen # 150 # assert f2.read() == "" @@ -190,13 +190,13 @@ def test(safefmode=False, kwargs={}): f2.write(" world!") f2.close() # 1) preserve mode and position - assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - assert f2mode == fmode - assert f2tell == ftell - # 3) prefer data over filehandle state - # assert open(fname).read() == "h\x00\x00\x00\x00 world!" - # assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? + # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" + # assert f2mode == fmode # assert f2tell == ftell + # 3) prefer data over filehandle state + assert open(fname).read() == "h world!" + assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? + assert f2tell == _ftell # 2) treat as if new filehandle, will truncate file # assert open(fname).read() == " world!" # assert f2mode == fmode @@ -244,7 +244,7 @@ def test(safefmode=False, kwargs={}): # 1) preserve mode and position # also 3) # position of writes cannot be changed on some OSs assert open(fname).read() == "h world!" - assert f2tell == ftell + assert f2tell == _ftell # 2) treat as if new filehandle, will seek(EOF) # assert open(fname).read() == "h world!" # assert f2tell == _ftell @@ -279,17 +279,18 @@ def test(safefmode=False, kwargs={}): f2 = dill.loads(f_dumped) assert f2.mode == fmode # 1) preserve mode and position #XXX: ? - assert f2.tell() == ftell # 200 - assert f2.read() == "" - f2.seek(0) - assert f2.read() == "" - assert f2.tell() == 0 - # 3) prefer data over filehandle state # assert f2.tell() == ftell # 200 # assert f2.read() == "" # f2.seek(0) # assert f2.read() == "" # assert f2.tell() == 0 + # 3) prefer data over filehandle state + # FIXME: this fails on systems where f2.tell() always returns 0 + # assert f2.tell() == ftell # 200 + assert f2.read() == "" + f2.seek(0) + assert f2.read() == "" + assert f2.tell() == 0 # 5) pickle data along with filehandle # assert f2.tell() == ftell # 200 # assert f2.read() == "" @@ -330,13 +331,13 @@ def test(safefmode=False, kwargs={}): f2.write(" world!") f2.close() # 1) preserve mode and position - assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - assert f2mode == fmode - assert f2tell == ftell - # 3) prefer data over filehandle state # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - # assert f2mode == 'w+' + # assert f2mode == fmode # assert f2tell == ftell + # 3) prefer data over filehandle state + assert open(fname).read() == " world!" + assert f2mode == 'w+' + assert f2tell == 0 # 2) treat as if new filehandle, will truncate file # assert open(fname).read() == " world!" # assert f2mode == fmode @@ -378,7 +379,7 @@ def test(safefmode=False, kwargs={}): assert f2mode == fmode # 1) preserve mode and position #XXX: also 3) assert open(fname).read() == " world!" # 3) - assert f2tell == ftell + assert f2tell == 0 # 2) treat as if new filehandle, will seek(EOF) # assert open(fname).read() == " world!" # assert f2tell == 0 @@ -411,17 +412,17 @@ def test(safefmode=False, kwargs={}): f2 = dill.loads(f_dumped) assert f2.mode == fmode # 1) preserve mode and position #XXX: ? - assert f2.tell() == ftell # 200 - assert f2.read() == _fstr[ftell:] - f2.seek(0) - assert f2.read() == _fstr - assert f2.tell() == _flen # 250 - # 3) prefer data over filehandle state # assert f2.tell() == ftell # 200 # assert f2.read() == _fstr[ftell:] # f2.seek(0) # assert f2.read() == _fstr # assert f2.tell() == _flen # 250 + # 3) prefer data over filehandle state + assert f2.tell() == ftell # 200 + assert f2.read() == _fstr[ftell:] + f2.seek(0) + assert f2.read() == _fstr + assert f2.tell() == _flen # 250 # 4) preserve mode and position, seek(EOF) if ftell > EOF # assert f2.tell() == ftell # 200 # assert f2.read() == _fstr[ftell:] @@ -463,13 +464,13 @@ def test(safefmode=False, kwargs={}): f2.write(" world!") f2.close() # 1) preserve mode and position - assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - assert f2mode == fmode - assert f2tell == ftell - # 3) prefer data over filehandle state - # assert open(fname).read() == "hello world!odbye!" - # assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? + # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" + # assert f2mode == fmode # assert f2tell == ftell + # 3) prefer data over filehandle state + assert open(fname).read() == "hello world!odbye!" + assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? + assert f2tell == ftell # 2) treat as if new filehandle, will truncate file # assert open(fname).read() == " world!" # assert f2mode == fmode From a7fbbcf5112afade90cf5dbf0a71ca90fed07b10 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Thu, 14 Aug 2014 11:32:39 +0100 Subject: [PATCH 04/11] Add different file pickling modes --- dill/__init__.py | 3 +- dill/dill.py | 119 +++++++----- tests/test_file.py | 445 +++++++++++++++++++-------------------------- 3 files changed, 265 insertions(+), 302 deletions(-) diff --git a/dill/__init__.py b/dill/__init__.py index bd3289f0..5f8a9940 100644 --- a/dill/__init__.py +++ b/dill/__init__.py @@ -26,7 +26,8 @@ from .dill import dump, dumps, load, loads, dump_session, load_session, \ Pickler, Unpickler, register, copy, pickle, pickles, HIGHEST_PROTOCOL, \ DEFAULT_PROTOCOL, PicklingError, UnpicklingError, \ - _revert_extension as revert_extension, _extend as extend + _revert_extension as revert_extension, _extend as extend, FMODE_NEWHANDLE, \ + FMODE_PRESERVEDATA, FMODE_PICKLECONTENTS from . import source, temp, detect # make sure "trace" is turned off diff --git a/dill/dill.py b/dill/dill.py index 6a3fe137..2a6aa5bc 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -133,12 +133,23 @@ def ndarrayinstance(obj): return False ExitType = type(exit) singletontypes = [] +### File modes +# pickles the file handle, preserving mode, with position of the unpickled +# object as for a new file handle. +FMODE_NEWHANDLE = 0 +# preserves existing data or create file if is does not exist, with +# position = min(pickled position, EOF), and mode which preserves behaviour +FMODE_PRESERVEDATA = 1 +# pickles the file handle, preserving mode and position, as well as the file +# contents +FMODE_PICKLECONTENTS = 2 + ### Shorthands (modified from python2.5/lib/pickle.py) def copy(obj, *args, **kwds): """use pickling to 'copy' an object""" return loads(dumps(obj, *args, **kwds)) -def dump(obj, file, protocol=None, byref=False, safe_file=False): +def dump(obj, file, protocol=None, byref=False, file_mode=FMODE_NEWHANDLE, safe_file=False): """pickle an object to a file""" if protocol is None: protocol = DEFAULT_PROTOCOL pik = Pickler(file, protocol) @@ -146,6 +157,7 @@ def dump(obj, file, protocol=None, byref=False, safe_file=False): _byref = pik._byref pik._byref = bool(byref) pik._safe_file = safe_file + pik._file_mode = file_mode # hack to catch subclassed numpy array instances if NumpyArrayType and ndarrayinstance(obj): @register(type(obj)) @@ -160,10 +172,10 @@ def save_numpy_array(pickler, obj): pik._byref = _byref return -def dumps(obj, protocol=None, byref=False, safe_file=False): +def dumps(obj, protocol=None, byref=False, file_mode=FMODE_NEWHANDLE, safe_file=False): """pickle an object to a string""" file = StringIO() - dump(obj, file, protocol, byref, safe_file) + dump(obj, file, protocol, byref, file_mode, safe_file) return file.getvalue() def load(file): @@ -233,6 +245,7 @@ class Pickler(StockPickler): _session = False _byref = False _safe_file = False + _file_mode = FMODE_NEWHANDLE pass def __init__(self, *args, **kwargs): @@ -357,7 +370,7 @@ def _create_lock(locked, *args): raise UnpicklingError("Cannot acquire lock") return lock -def _create_filehandle(name, mode, position, closed, open=open, safe=False): # buffering=0 +def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdata): # buffering=0 # only pickles the handle, not the file contents... good? or StringIO(data)? # (for file contents see: http://effbot.org/librarybook/copy-reg.htm) # NOTE: handle special cases first (are there more special cases?) @@ -373,43 +386,57 @@ def _create_filehandle(name, mode, position, closed, open=open, safe=False): # b f = tempfile.TemporaryFile(mode) else: import os - # Mode translation - # Mode | Unpickled mode - # --------|--------------- - # r | r - # r+ | r+ - # w | r+ - # w+ | r+ - # a | a - # a+ | a+ - # Note: If the file does not exist, the mode is not translated if mode == "x": mode = "w" - if os.path.exists(name): + if not os.path.exists(name): + if safe: + raise IOError("File '%s' does not exist" % name) + elif "r" in mode and file_mode != FMODE_PICKLECONTENTS: + name = os.devnull + current_size = 0 + else: + current_size = os.path.getsize(name) + + if file_mode == FMODE_PRESERVEDATA and os.path.exists(name): + # Mode translation + # Mode | Unpickled mode + # --------|--------------- + # r | r + # r+ | r+ + # w | r+ + # w+ | r+ + # a | a + # a+ | a+ + # Note: If the file does not exist, the mode is not translated mode = mode.replace("w+", "r+") mode = mode.replace("w", "r+") - elif safe: - raise IOError("File '%s' does not exist" % name) - elif "r" in mode: - name = os.devnull - if safe: - if position > os.path.getsize(name): + + if position > current_size: + if safe: raise IOError("File '%s' is too short" % name) + elif file_mode == FMODE_PRESERVEDATA: + position = current_size # try to open the file by name # NOTE: has different fileno try: - f = open(name, mode) #FIXME: missing: *buffering*, encoding, softspace + if file_mode == FMODE_PICKLECONTENTS: + f = open(name, mode if "w" in mode else "w") + f.write(fdata) + if "w" not in mode: + f.close() + f = open(name, mode) #FIXME: missing: *buffering*, encoding, softspace + else: + f = open(name, mode) #FIXME: missing: *buffering*, encoding, softspace except IOError: err = sys.exc_info()[1] raise UnpicklingError(err) #XXX: python default is closed '' file/mode if closed: f.close() - elif position >= 0: - eof = os.path.getsize(name) - f.seek(position if position < eof else eof) + elif position >= 0 and file_mode != FMODE_NEWHANDLE: + f.seek(position) return f def _create_stringi(value, position, closed): @@ -616,15 +643,8 @@ def save_attrgetter(pickler, obj): pickler.save_reduce(type(obj), tuple(attrs), obj=obj) return -# __getstate__ explicitly added to raise TypeError when pickling: -# http://www.gossamer-threads.com/lists/python/bugs/871199 -@register(FileType) #XXX: in 3.x has buffer=0, needs different _create? -@register(BufferedRandomType) -@register(BufferedReaderType) -@register(BufferedWriterType) -@register(TextWrapperType) -def save_file(pickler, obj): - log.info("Fi: %s" % obj) +def _save_file(pickler, obj, open_): + obj.flush() if obj.closed: position = None else: @@ -632,24 +652,35 @@ def save_file(pickler, obj): position = -1 else: position = obj.tell() - pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, \ - obj.closed, open, pickler._safe_file), obj=obj) + if pickler._file_mode == FMODE_PICKLECONTENTS: + f = open_(obj.name, "r") + fdata = f.read() + f.close() + else: + fdata = "" + pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, + obj.closed, open_, pickler._safe_file, + pickler._file_mode, fdata), obj=obj) return -if PyTextWrapperType: #XXX: are stdout, stderr or stdin ever _pyio files? + +@register(FileType) #XXX: in 3.x has buffer=0, needs different _create? +@register(BufferedRandomType) +@register(BufferedReaderType) +@register(BufferedWriterType) +@register(TextWrapperType) +def save_file(pickler, obj): + log.info("Fi: %s" % obj) + return _save_file(pickler, obj, open) + +if PyTextWrapperType: @register(PyBufferedRandomType) @register(PyBufferedReaderType) @register(PyBufferedWriterType) @register(PyTextWrapperType) def save_file(pickler, obj): log.info("Fi: %s" % obj) - if obj.closed: - position = None - else: - position = obj.tell() - pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, \ - obj.closed, _open, pickler._safe_file), obj=obj) - return + return _save_file(pickler, obj, _open) # The following two functions are based on 'saveCStringIoInput' # and 'saveCStringIoOutput' from spickle diff --git a/tests/test_file.py b/tests/test_file.py index 3fd03d14..1fc5ce35 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -1,4 +1,4 @@ -#usr/bin/env python +# usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2008-2014 California Institute of Technology. @@ -36,14 +36,14 @@ def throws(op, args, exc): return False -def test(safefmode=False, kwargs={}): +def test(safe_file, file_mode): # file exists, with same contents # read write_randomness() f = open(fname, "r") - _f = dill.loads(dill.dumps(f, **kwargs)) + _f = dill.loads(dill.dumps(f, safe_file=safe_file, file_mode=file_mode)) assert _f.mode == f.mode assert _f.tell() == f.tell() assert _f.read() == f.read() @@ -54,7 +54,7 @@ def test(safefmode=False, kwargs={}): f = open(fname, "w") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -64,26 +64,20 @@ def test(safefmode=False, kwargs={}): f2.write(" world!") f2.close() - # 1) preserve mode and position - # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 2) treat as if new filehandle, will truncate file - # assert open(fname).read() == " world!" - # assert f2mode == fmode - # assert f2tell == 0 - # 3) prefer data over filehandle state - assert open(fname).read() == "hello world!" - assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? - assert f2tell == ftell - # 4) use "r" to read data, then use "w" to write new file - # assert open(fname).read() == "hello world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2mode == fmode - # assert f2tell == ftell + if file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == " world!" + assert f2mode == fmode + assert f2tell == 0 + elif file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == "hello world!" + assert f2mode == 'r+' + assert f2tell == ftell + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2mode == fmode + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") # append @@ -91,7 +85,7 @@ def test(safefmode=False, kwargs={}): f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -102,18 +96,17 @@ def test(safefmode=False, kwargs={}): f2.close() assert f2mode == fmode - # 1) preserve mode and position # also 3) - assert open(fname).read() == "hello world!" - assert f2tell == ftell - # 2) treat as if new filehandle, will seek(EOF) - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell - # 4) use "r" to read data, then use "a" to write new file - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") # file exists, with different contents (smaller size) # read @@ -122,46 +115,36 @@ def test(safefmode=False, kwargs={}): f = open(fname, "r") fstr = f.read() - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() _flen = 150 _fstr = write_randomness(number=_flen) - if safefmode: # throw error if ftell > EOF + if safe_file: # throw error if ftell > EOF assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) assert f2.mode == fmode - # 1) preserve mode and position #XXX: ? - # assert f2.tell() == ftell # 200 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 150 - # 3) prefer data over filehandle state - assert f2.tell() == _flen - assert f2.read() == "" - f2.seek(0) - assert f2.read() == _fstr - assert f2.tell() == _flen # 150 - # 4) preserve mode and position, seek(EOF) if ftell > EOF - # assert f2.tell() == _flen # 150 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 150 - # 2) treat as if new filehandle, will seek(0) - # assert f2.tell() == 0 - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 150 - # 5) pickle data along with filehandle - # assert f2.tell() == ftell # 200 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == fstr - # assert f2.tell() == ftell # 200 + if file_mode == dill.FMODE_PRESERVEDATA: + assert f2.tell() == _flen + assert f2.read() == "" + f2.seek(0) + assert f2.read() == _fstr + assert f2.tell() == _flen # 150 + elif file_mode == dill.FMODE_NEWHANDLE: + assert f2.tell() == 0 + assert f2.read() == _fstr + assert f2.tell() == _flen # 150 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert f2.tell() == ftell # 200 + assert f2.read() == "" + f2.seek(0) + assert f2.read() == fstr + assert f2.tell() == ftell # 200 + else: + raise RuntimeError("Uncovered file mode!") f2.close() # write @@ -170,7 +153,7 @@ def test(safefmode=False, kwargs={}): f = open(fname, "w") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -181,7 +164,7 @@ def test(safefmode=False, kwargs={}): _ftell = f.tell() f.close() - if safefmode: # throw error if ftell > EOF + if safe_file: # throw error if ftell > EOF assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -189,39 +172,29 @@ def test(safefmode=False, kwargs={}): f2tell = f2.tell() f2.write(" world!") f2.close() - # 1) preserve mode and position - # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 3) prefer data over filehandle state - assert open(fname).read() == "h world!" - assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? - assert f2tell == _ftell - # 2) treat as if new filehandle, will truncate file - # assert open(fname).read() == " world!" - # assert f2mode == fmode - # assert f2tell == 0 - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 4a) use "r" to read data, then use "w" to write new file - # assert open(fname).read() == "h\x00\x00\x00\x00 world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 4b) preserve mode and position, seek(EOF) if ftell > EOF - # assert open(fname).read() == "h world!" - # assert f2mode == fmode - # assert f2tell == _ftell + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == "h world!" + assert f2mode == 'r+' + assert f2tell == _ftell + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == " world!" + assert f2mode == fmode + assert f2tell == 0 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2mode == fmode + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") f2.close() # append - write_randomness() + trunc_file() f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -232,7 +205,7 @@ def test(safefmode=False, kwargs={}): _ftell = f.tell() f.close() - if safefmode: # throw error if ftell > EOF + if safe_file: # throw error if ftell > EOF assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -241,22 +214,18 @@ def test(safefmode=False, kwargs={}): f2.write(" world!") f2.close() assert f2mode == fmode - # 1) preserve mode and position # also 3) - # position of writes cannot be changed on some OSs - assert open(fname).read() == "h world!" - assert f2tell == _ftell - # 2) treat as if new filehandle, will seek(EOF) - # assert open(fname).read() == "h world!" - # assert f2tell == _ftell - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell - # 4a) use "r" to read data, then use "a" to write new file - # assert open(fname).read() == "h world!" - # assert f2tell == ftell - # 4b) preserve mode and position, seek(EOF) if ftell > EOF - # assert open(fname).read() == "h world!" - # assert f2tell == _ftell + if file_mode == dill.FMODE_PRESERVEDATA: + # position of writes cannot be changed on some OSs + assert open(fname).read() == "h world!" + assert f2tell == _ftell + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == "h world!" + assert f2tell == _ftell + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") f2.close() # file does not exist @@ -266,47 +235,37 @@ def test(safefmode=False, kwargs={}): f = open(fname, "r") fstr = f.read() - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() os.remove(fname) - if safefmode: # throw error if file DNE + if safe_file: # throw error if file DNE assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) assert f2.mode == fmode - # 1) preserve mode and position #XXX: ? - # assert f2.tell() == ftell # 200 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == "" - # assert f2.tell() == 0 - # 3) prefer data over filehandle state - # FIXME: this fails on systems where f2.tell() always returns 0 - # assert f2.tell() == ftell # 200 - assert f2.read() == "" - f2.seek(0) - assert f2.read() == "" - assert f2.tell() == 0 - # 5) pickle data along with filehandle - # assert f2.tell() == ftell # 200 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == fstr - # assert f2.tell() == ftell # 200 - # 2) treat as if new filehandle, will seek(0) - # assert f2.tell() == 0 - # assert f2.read() == "" - # assert f2.tell() == 0 - # 4) preserve mode and position, seek(EOF) if ftell > EOF - # assert f2.tell() == 0 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == "" - # assert f2.tell() == 0 + if file_mode == dill.FMODE_PRESERVEDATA: + # FIXME: this fails on systems where f2.tell() always returns 0 + # assert f2.tell() == ftell # 200 + assert f2.read() == "" + f2.seek(0) + assert f2.read() == "" + assert f2.tell() == 0 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert f2.tell() == ftell # 200 + assert f2.read() == "" + f2.seek(0) + assert f2.read() == fstr + assert f2.tell() == ftell # 200 + elif file_mode == dill.FMODE_NEWHANDLE: + assert f2.tell() == 0 + assert f2.read() == "" + assert f2.tell() == 0 + else: + raise RuntimeError("Uncovered file mode!") f2.close() # write @@ -315,14 +274,14 @@ def test(safefmode=False, kwargs={}): f = open(fname, "w+") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) ftell = f.tell() fmode = f.mode f.close() os.remove(fname) - if safefmode: # throw error if file DNE + if safe_file: # throw error if file DNE assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -330,30 +289,20 @@ def test(safefmode=False, kwargs={}): f2tell = f2.tell() f2.write(" world!") f2.close() - # 1) preserve mode and position - # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 3) prefer data over filehandle state - assert open(fname).read() == " world!" - assert f2mode == 'w+' - assert f2tell == 0 - # 2) treat as if new filehandle, will truncate file - # assert open(fname).read() == " world!" - # assert f2mode == fmode - # assert f2tell == 0 - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 4a) use "r" to read data, then use "w" to write new file - # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 4b) preserve mode and position, seek(EOF) if ftell > EOF - # assert open(fname).read() == " world!" - # assert f2mode == fmode - # assert f2tell == 0 + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == " world!" + assert f2mode == 'w+' + assert f2tell == 0 + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == " world!" + assert f2mode == fmode + assert f2tell == 0 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2mode == fmode + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") # append @@ -361,14 +310,14 @@ def test(safefmode=False, kwargs={}): f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) ftell = f.tell() fmode = f.mode f.close() os.remove(fname) - if safefmode: # throw error if file DNE + if safe_file: # throw error if file DNE assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -377,21 +326,17 @@ def test(safefmode=False, kwargs={}): f2.write(" world!") f2.close() assert f2mode == fmode - # 1) preserve mode and position #XXX: also 3) - assert open(fname).read() == " world!" # 3) - assert f2tell == 0 - # 2) treat as if new filehandle, will seek(EOF) - # assert open(fname).read() == " world!" - # assert f2tell == 0 - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell - # 4a) use "r" to read data, then use "a" to write new file - # assert open(fname).read() == " world!" - # assert f2tell == ftell - # 4b) preserve mode and position, seek(EOF) if ftell > EOF - # assert open(fname).read() == " world!" - # assert f2tell == 0 + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == " world!" + assert f2tell == 0 + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == " world!" + assert f2tell == 0 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") # file exists, with different contents (larger size) # read @@ -400,89 +345,72 @@ def test(safefmode=False, kwargs={}): f = open(fname, "r") fstr = f.read() - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() _flen = 250 _fstr = write_randomness(number=_flen) - #XXX: no safefmode: no way to be 'safe'? + # XXX: no safe_file: no way to be 'safe'? f2 = dill.loads(f_dumped) assert f2.mode == fmode - # 1) preserve mode and position #XXX: ? - # assert f2.tell() == ftell # 200 - # assert f2.read() == _fstr[ftell:] - # f2.seek(0) - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 250 - # 3) prefer data over filehandle state - assert f2.tell() == ftell # 200 - assert f2.read() == _fstr[ftell:] - f2.seek(0) - assert f2.read() == _fstr - assert f2.tell() == _flen # 250 - # 4) preserve mode and position, seek(EOF) if ftell > EOF - # assert f2.tell() == ftell # 200 - # assert f2.read() == _fstr[ftell:] - # f2.seek(0) - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 250 - # 2) treat as if new filehandle, will seek(0) - # assert f2.tell() == 0 - # assert f2.read() == _fstr - # assert f2.tell() == _flen # 250 - # 5) pickle data along with filehandle - # assert f2.tell() == ftell # 200 - # assert f2.read() == "" - # f2.seek(0) - # assert f2.read() == fstr - # assert f2.tell() == ftell # 200 - f2.close() #XXX: other alternatives? + if file_mode == dill.FMODE_PRESERVEDATA: + assert f2.tell() == ftell # 200 + assert f2.read() == _fstr[ftell:] + f2.seek(0) + assert f2.read() == _fstr + assert f2.tell() == _flen # 250 + elif file_mode == dill.FMODE_NEWHANDLE: + assert f2.tell() == 0 + assert f2.read() == _fstr + assert f2.tell() == _flen # 250 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert f2.tell() == ftell # 200 + assert f2.read() == "" + f2.seek(0) + assert f2.read() == fstr + assert f2.tell() == ftell # 200 + else: + raise RuntimeError("Uncovered file mode!") + f2.close() # XXX: other alternatives? # write f = open(fname, "w") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) fmode = f.mode ftell = f.tell() -# f.close() + fstr = open(fname).read() -# f = open(fname, "a") f.write(" and goodbye!") _ftell = f.tell() f.close() - #XXX: no safefmode: no way to be 'safe'? + # XXX: no safe_file: no way to be 'safe'? f2 = dill.loads(f_dumped) f2mode = f2.mode f2tell = f2.tell() f2.write(" world!") f2.close() - # 1) preserve mode and position - # assert open(fname).read() == "\x00\x00\x00\x00\x00 world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 3) prefer data over filehandle state - assert open(fname).read() == "hello world!odbye!" - assert f2mode == 'r+' #XXX: have to decide 'r+', 'a', ...? - assert f2tell == ftell - # 2) treat as if new filehandle, will truncate file - # assert open(fname).read() == " world!" - # assert f2mode == fmode - # assert f2tell == 0 - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2mode == fmode - # assert f2tell == ftell - # 4) use "r" to read data, then use "w" to write new file - # assert open(fname).read() == "hello world!odbye!" - # assert f2mode == fmode - # assert f2tell == ftell + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == "hello world!odbye!" + assert f2mode == 'r+' # XXX: have to decide 'r+', 'a', ...? + assert f2tell == ftell + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == " world!" + assert f2mode == fmode + assert f2tell == 0 + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2mode == fmode + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") f2.close() # append @@ -491,7 +419,7 @@ def test(safefmode=False, kwargs={}): f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, **kwargs) + f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) fmode = f.mode ftell = f.tell() fstr = open(fname).read() @@ -500,7 +428,7 @@ def test(safefmode=False, kwargs={}): _ftell = f.tell() f.close() - #XXX: no safefmode: no way to be 'safe'? + # XXX: no safe_file: no way to be 'safe'? f2 = dill.loads(f_dumped) f2mode = f2.mode @@ -508,23 +436,26 @@ def test(safefmode=False, kwargs={}): f2.write(" world!") f2.close() assert f2mode == fmode - # 1) preserve mode and position # also 3) - assert open(fname).read() == "hello and goodbye! world!" - assert f2tell == ftell - # 2) treat as if new filehandle, will seek(EOF) - # assert open(fname).read() == "hello and goodbye! world!" - # assert f2tell == _ftell - # 5) pickle data along with filehandle - # assert open(fname).read() == "hello world!" - # assert f2tell == ftell - # 4) use "r" to read data, then use "a" to write new file - # assert open(fname).read() == "hello and goodbye! world!" - # assert f2tell == ftell + if file_mode == dill.FMODE_PRESERVEDATA: + assert open(fname).read() == "hello and goodbye! world!" + assert f2tell == ftell + elif file_mode == dill.FMODE_NEWHANDLE: + assert open(fname).read() == "hello and goodbye! world!" + assert f2tell == _ftell + elif file_mode == dill.FMODE_PICKLECONTENTS: + assert open(fname).read() == "hello world!" + assert f2tell == ftell + else: + raise RuntimeError("Uncovered file mode!") f2.close() -test() +test(safe_file=False, file_mode=dill.FMODE_NEWHANDLE) +test(safe_file=False, file_mode=dill.FMODE_PRESERVEDATA) +test(safe_file=False, file_mode=dill.FMODE_PICKLECONTENTS) # TODO: switch this on when #57 is closed -# test(True, {"safe_file": True}) +# test(safe_file=True, file_mode=dill.FMODE_NEWHANDLE) +# test(safe_file=True, file_mode=dill.FMODE_PRESERVEDATA) +# test(safe_file=True, file_mode=dill.FMODE_PICKLECONTENTS) if os.path.exists(fname): os.remove(fname) From 087c00899ef55f31d36e7aee51a958b17daf8c91 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Thu, 14 Aug 2014 15:05:41 +0100 Subject: [PATCH 05/11] Change to use os.open to avoid visible mode change --- dill/dill.py | 25 ++++++++----------------- tests/test_file.py | 8 +++++--- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index 2a6aa5bc..b525c80d 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -399,20 +399,6 @@ def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdat else: current_size = os.path.getsize(name) - if file_mode == FMODE_PRESERVEDATA and os.path.exists(name): - # Mode translation - # Mode | Unpickled mode - # --------|--------------- - # r | r - # r+ | r+ - # w | r+ - # w+ | r+ - # a | a - # a+ | a+ - # Note: If the file does not exist, the mode is not translated - mode = mode.replace("w+", "r+") - mode = mode.replace("w", "r+") - if position > current_size: if safe: raise IOError("File '%s' is too short" % name) @@ -421,18 +407,23 @@ def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdat # try to open the file by name # NOTE: has different fileno try: + #FIXME: missing: *buffering*, encoding, softspace if file_mode == FMODE_PICKLECONTENTS: f = open(name, mode if "w" in mode else "w") f.write(fdata) if "w" not in mode: f.close() - f = open(name, mode) #FIXME: missing: *buffering*, encoding, softspace + f = open(name, mode) + elif file_mode == FMODE_PRESERVEDATA and "w" in mode: + # stop truncation when opening + def opener(file, flags): + return os.open(file, flags ^ os.O_TRUNC) + f = open(name, mode, opener=opener) else: - f = open(name, mode) #FIXME: missing: *buffering*, encoding, softspace + f = open(name, mode) except IOError: err = sys.exc_info()[1] raise UnpicklingError(err) - #XXX: python default is closed '' file/mode if closed: f.close() elif position >= 0 and file_mode != FMODE_NEWHANDLE: diff --git a/tests/test_file.py b/tests/test_file.py index 1fc5ce35..e3294fac 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -61,6 +61,7 @@ def test(safe_file, file_mode): f2 = dill.loads(f_dumped) f2mode = f2.mode f2tell = f2.tell() + f2name = f2.name f2.write(" world!") f2.close() @@ -70,8 +71,9 @@ def test(safe_file, file_mode): assert f2tell == 0 elif file_mode == dill.FMODE_PRESERVEDATA: assert open(fname).read() == "hello world!" - assert f2mode == 'r+' + assert f2mode == fmode assert f2tell == ftell + assert f2name == fname elif file_mode == dill.FMODE_PICKLECONTENTS: assert open(fname).read() == "hello world!" assert f2mode == fmode @@ -174,7 +176,7 @@ def test(safe_file, file_mode): f2.close() if file_mode == dill.FMODE_PRESERVEDATA: assert open(fname).read() == "h world!" - assert f2mode == 'r+' + assert f2mode == fmode assert f2tell == _ftell elif file_mode == dill.FMODE_NEWHANDLE: assert open(fname).read() == " world!" @@ -399,7 +401,7 @@ def test(safe_file, file_mode): f2.close() if file_mode == dill.FMODE_PRESERVEDATA: assert open(fname).read() == "hello world!odbye!" - assert f2mode == 'r+' # XXX: have to decide 'r+', 'a', ...? + assert f2mode == fmode assert f2tell == ftell elif file_mode == dill.FMODE_NEWHANDLE: assert open(fname).read() == " world!" From 47cc8b0f1b8e895b9b4681cdb260daf84c9097f1 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Thu, 14 Aug 2014 18:14:27 +0100 Subject: [PATCH 06/11] Add file modes to `__all__`, switch on safe_file tests, improve mode x handling and fix shabang --- dill/dill.py | 16 +++++++--------- tests/test_file.py | 11 ++++++----- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index b525c80d..03a0094d 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -14,10 +14,11 @@ Test against "all" python types (Std. Lib. CH 1-15 @ 2.7) by mmckerns. Test against CH16+ Std. Lib. ... TBD. """ -__all__ = ['dump','dumps','load','loads','dump_session','load_session',\ - 'Pickler','Unpickler','register','copy','pickle','pickles',\ - 'HIGHEST_PROTOCOL','DEFAULT_PROTOCOL',\ - 'PicklingError','UnpicklingError'] +__all__ = ['dump','dumps','load','loads','dump_session','load_session', + 'Pickler','Unpickler','register','copy','pickle','pickles', + 'HIGHEST_PROTOCOL','DEFAULT_PROTOCOL', + 'PicklingError','UnpicklingError','FMODE_NEWHANDLE', + 'FMODE_PRESERVEDATA','FMODE_PICKLECONTENTS'] import logging log = logging.getLogger("dill") @@ -379,16 +380,13 @@ def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdat if name in list(names.keys()): f = names[name] #XXX: safer "f=sys.stdin" elif name == '': - import os f = os.tmpfile() elif name == '': import tempfile f = tempfile.TemporaryFile(mode) else: - import os - - if mode == "x": - mode = "w" + # treat x mode as w mode + mode = mode.replace("x", "w") if not os.path.exists(name): if safe: diff --git a/tests/test_file.py b/tests/test_file.py index e3294fac..c0252711 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -1,4 +1,4 @@ -# usr/bin/env python +#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2008-2014 California Institute of Technology. @@ -455,9 +455,10 @@ def test(safe_file, file_mode): test(safe_file=False, file_mode=dill.FMODE_NEWHANDLE) test(safe_file=False, file_mode=dill.FMODE_PRESERVEDATA) test(safe_file=False, file_mode=dill.FMODE_PICKLECONTENTS) -# TODO: switch this on when #57 is closed -# test(safe_file=True, file_mode=dill.FMODE_NEWHANDLE) -# test(safe_file=True, file_mode=dill.FMODE_PRESERVEDATA) -# test(safe_file=True, file_mode=dill.FMODE_PICKLECONTENTS) + +test(safe_file=True, file_mode=dill.FMODE_NEWHANDLE) +test(safe_file=True, file_mode=dill.FMODE_PRESERVEDATA) +test(safe_file=True, file_mode=dill.FMODE_PICKLECONTENTS) + if os.path.exists(fname): os.remove(fname) From 0a2fa3139951a8ef43377413e646f98ebe8907a8 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Mon, 18 Aug 2014 21:08:51 +0100 Subject: [PATCH 07/11] Add backwards compatibility to file handling --- dill/dill.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index 03a0094d..c8d76d5b 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -414,9 +414,34 @@ def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdat f = open(name, mode) elif file_mode == FMODE_PRESERVEDATA and "w" in mode: # stop truncation when opening - def opener(file, flags): - return os.open(file, flags ^ os.O_TRUNC) - f = open(name, mode, opener=opener) + flags = os.O_CREAT + if "+" in mode: + flags |= os.O_RDWR + else: + flags |= os.O_WRONLY + f = os.fdopen(os.open(name, flags), mode) + # set name to the correct value + if PY3: + r = getattr(f, "buffer", f) + r = getattr(r, "raw", r) + r.name = name + else: + class FILE(ctypes.Structure): + _fields_ = [("refcount", ctypes.c_long), + ("type_obj", ctypes.py_object), + ("file_pointer", ctypes.c_voidp), + ("name", ctypes.py_object)] + + class PyObject(ctypes.Structure): + _fields_ = [ + ("ob_refcnt", ctypes.c_int), + ("ob_type", ctypes.py_object) + ] + if not HAS_CTYPES: + raise RuntimeError("Need ctypes to set file name") + ctypes.cast(id(f), ctypes.POINTER(FILE)).contents.name = name + ctypes.cast(id(name), ctypes.POINTER(PyObject)).contents.ob_refcnt += 1 + assert f.name == name else: f = open(name, mode) except IOError: From 4edc3c0b5e181023d6784f1b39d995a51b196fb3 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Wed, 20 Aug 2014 20:21:12 +0100 Subject: [PATCH 08/11] Fix for python 2.5 --- tests/test_file.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_file.py b/tests/test_file.py index c0252711..8e387ad2 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -15,11 +15,13 @@ def write_randomness(number=200): - with open(fname, "w") as f: - for i in range(number): - f.write(random.choice(rand_chars)) - with open(fname, "r") as f: - contents = f.read() + f = open(fname, "w") + for i in range(number): + f.write(random.choice(rand_chars)) + f.close() + f = open(fname, "r") + contents = f.read() + f.close() return contents From b435bb7b1cd892ca0446e190d06088424795c278 Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Wed, 20 Aug 2014 21:19:39 +0100 Subject: [PATCH 09/11] Add extra handling for x mode --- dill/dill.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index c8d76d5b..2cfdcb56 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -386,7 +386,8 @@ def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdat f = tempfile.TemporaryFile(mode) else: # treat x mode as w mode - mode = mode.replace("x", "w") + if "x" in mode and sys.hexversion < 0x03030000: + raise IOError("invalid mode 'x'") if not os.path.exists(name): if safe: @@ -412,7 +413,8 @@ def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdat if "w" not in mode: f.close() f = open(name, mode) - elif file_mode == FMODE_PRESERVEDATA and "w" in mode: + elif file_mode == FMODE_PRESERVEDATA \ + and "w" in mode or "x" in mode: # stop truncation when opening flags = os.O_CREAT if "+" in mode: From e256d57d5da03a3b4f1d39f36be813dd7cdf2c3c Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Thu, 21 Aug 2014 10:15:24 +0100 Subject: [PATCH 10/11] Change safe_file to a more generic name safeio --- dill/dill.py | 18 ++++++++--------- tests/test_file.py | 50 +++++++++++++++++++++++----------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index e094b841..201a6c7f 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -151,14 +151,14 @@ def copy(obj, *args, **kwds): """use pickling to 'copy' an object""" return loads(dumps(obj, *args, **kwds)) -def dump(obj, file, protocol=None, byref=False, file_mode=FMODE_NEWHANDLE, safe_file=False): +def dump(obj, file, protocol=None, byref=False, file_mode=FMODE_NEWHANDLE, safeio=False): """pickle an object to a file""" if protocol is None: protocol = DEFAULT_PROTOCOL pik = Pickler(file, protocol) pik._main_module = _main_module _byref = pik._byref pik._byref = bool(byref) - pik._safe_file = safe_file + pik._safeio = safeio pik._file_mode = file_mode # hack to catch subclassed numpy array instances if NumpyArrayType and ndarrayinstance(obj): @@ -174,10 +174,10 @@ def save_numpy_array(pickler, obj): pik._byref = _byref return -def dumps(obj, protocol=None, byref=False, file_mode=FMODE_NEWHANDLE, safe_file=False): +def dumps(obj, protocol=None, byref=False, file_mode=FMODE_NEWHANDLE, safeio=False): """pickle an object to a string""" file = StringIO() - dump(obj, file, protocol, byref, file_mode, safe_file) + dump(obj, file, protocol, byref, file_mode, safeio) return file.getvalue() def load(file): @@ -372,7 +372,7 @@ def _create_lock(locked, *args): raise UnpicklingError("Cannot acquire lock") return lock -def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdata): # buffering=0 +def _create_filehandle(name, mode, position, closed, open, safeio, file_mode, fdata): # buffering=0 # only pickles the handle, not the file contents... good? or StringIO(data)? # (for file contents see: http://effbot.org/librarybook/copy-reg.htm) # NOTE: handle special cases first (are there more special cases?) @@ -391,7 +391,7 @@ def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdat raise IOError("invalid mode 'x'") if not os.path.exists(name): - if safe: + if safeio: raise IOError("File '%s' does not exist" % name) elif "r" in mode and file_mode != FMODE_PICKLECONTENTS: name = os.devnull @@ -400,7 +400,7 @@ def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdat current_size = os.path.getsize(name) if position > current_size: - if safe: + if safeio: raise IOError("File '%s' is too short" % name) elif file_mode == FMODE_PRESERVEDATA: position = current_size @@ -415,7 +415,7 @@ def _create_filehandle(name, mode, position, closed, open, safe, file_mode, fdat f.close() f = open(name, mode) elif file_mode == FMODE_PRESERVEDATA \ - and "w" in mode or "x" in mode: + and ("w" in mode or "x" in mode): # stop truncation when opening flags = os.O_CREAT if "+" in mode: @@ -677,7 +677,7 @@ def _save_file(pickler, obj, open_): fdata = "" pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, obj.closed, open_, pickler._safe_file, - pickler._file_mode, fdata), obj=obj) + pickler._safeio, fdata), obj=obj) return diff --git a/tests/test_file.py b/tests/test_file.py index 8e387ad2..e0b5ad0a 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -38,14 +38,14 @@ def throws(op, args, exc): return False -def test(safe_file, file_mode): +def test(safeio, file_mode): # file exists, with same contents # read write_randomness() f = open(fname, "r") - _f = dill.loads(dill.dumps(f, safe_file=safe_file, file_mode=file_mode)) + _f = dill.loads(dill.dumps(f, safeio=safeio, file_mode=file_mode)) assert _f.mode == f.mode assert _f.tell() == f.tell() assert _f.read() == f.read() @@ -56,7 +56,7 @@ def test(safe_file, file_mode): f = open(fname, "w") f.write("hello") - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -89,7 +89,7 @@ def test(safe_file, file_mode): f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -119,14 +119,14 @@ def test(safe_file, file_mode): f = open(fname, "r") fstr = f.read() - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() _flen = 150 _fstr = write_randomness(number=_flen) - if safe_file: # throw error if ftell > EOF + if safeio: # throw error if ftell > EOF assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -157,7 +157,7 @@ def test(safe_file, file_mode): f = open(fname, "w") f.write("hello") - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -168,7 +168,7 @@ def test(safe_file, file_mode): _ftell = f.tell() f.close() - if safe_file: # throw error if ftell > EOF + if safeio: # throw error if ftell > EOF assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -198,7 +198,7 @@ def test(safe_file, file_mode): f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -209,7 +209,7 @@ def test(safe_file, file_mode): _ftell = f.tell() f.close() - if safe_file: # throw error if ftell > EOF + if safeio: # throw error if ftell > EOF assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -239,14 +239,14 @@ def test(safe_file, file_mode): f = open(fname, "r") fstr = f.read() - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() os.remove(fname) - if safe_file: # throw error if file DNE + if safeio: # throw error if file DNE assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -278,14 +278,14 @@ def test(safe_file, file_mode): f = open(fname, "w+") f.write("hello") - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) ftell = f.tell() fmode = f.mode f.close() os.remove(fname) - if safe_file: # throw error if file DNE + if safeio: # throw error if file DNE assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -314,14 +314,14 @@ def test(safe_file, file_mode): f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) ftell = f.tell() fmode = f.mode f.close() os.remove(fname) - if safe_file: # throw error if file DNE + if safeio: # throw error if file DNE assert throws(dill.loads, (f_dumped,), IOError) else: f2 = dill.loads(f_dumped) @@ -349,7 +349,7 @@ def test(safe_file, file_mode): f = open(fname, "r") fstr = f.read() - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() f.close() @@ -384,7 +384,7 @@ def test(safe_file, file_mode): f = open(fname, "w") f.write("hello") - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() @@ -423,7 +423,7 @@ def test(safe_file, file_mode): f = open(fname, "a") f.write("hello") - f_dumped = dill.dumps(f, safe_file=safe_file, file_mode=file_mode) + f_dumped = dill.dumps(f, safeio=safeio, file_mode=file_mode) fmode = f.mode ftell = f.tell() fstr = open(fname).read() @@ -454,13 +454,13 @@ def test(safe_file, file_mode): f2.close() -test(safe_file=False, file_mode=dill.FMODE_NEWHANDLE) -test(safe_file=False, file_mode=dill.FMODE_PRESERVEDATA) -test(safe_file=False, file_mode=dill.FMODE_PICKLECONTENTS) +test(safeio=False, file_mode=dill.FMODE_NEWHANDLE) +test(safeio=False, file_mode=dill.FMODE_PRESERVEDATA) +test(safeio=False, file_mode=dill.FMODE_PICKLECONTENTS) -test(safe_file=True, file_mode=dill.FMODE_NEWHANDLE) -test(safe_file=True, file_mode=dill.FMODE_PRESERVEDATA) -test(safe_file=True, file_mode=dill.FMODE_PICKLECONTENTS) +test(safeio=True, file_mode=dill.FMODE_NEWHANDLE) +test(safeio=True, file_mode=dill.FMODE_PRESERVEDATA) +test(safeio=True, file_mode=dill.FMODE_PICKLECONTENTS) if os.path.exists(fname): os.remove(fname) From fb87158da8e99a2cf3d15d22cb9862b7effd5a9a Mon Sep 17 00:00:00 2001 From: Matthew Joyce Date: Thu, 21 Aug 2014 10:24:00 +0100 Subject: [PATCH 11/11] Fix test failure (stupid mistake) --- dill/dill.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dill/dill.py b/dill/dill.py index 201a6c7f..4cd528cc 100644 --- a/dill/dill.py +++ b/dill/dill.py @@ -676,8 +676,8 @@ def _save_file(pickler, obj, open_): else: fdata = "" pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, - obj.closed, open_, pickler._safe_file, - pickler._safeio, fdata), obj=obj) + obj.closed, open_, pickler._safeio, + pickler._file_mode, fdata), obj=obj) return