Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Session: improvements to documentation + handle unpickleable objects #527

Open
wants to merge 33 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 32 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
ab13325
code formatting changes
leogama Jul 16, 2022
33ca2ed
remove unnecessary '_main_modified' attribute from pickler
leogama Jul 16, 2022
ad8db21
new _open() function to handle file names and file-like objects
leogama Jul 16, 2022
da4cc07
merge function _make_peekable() with _open()
leogama Jul 16, 2022
1732e3d
new function is_module_pickle()
leogama Jul 16, 2022
2fdd31d
move session-related code to session.py submodule
leogama Jul 16, 2022
6b55755
session: deal with modules with unpickleable objects
leogama Jul 19, 2022
aac47b5
disable framing when using the 'refonfail' option
leogama Jul 19, 2022
5e4d912
rename is_module_pickle() to is_pickled_module(); fix _dill's __all__
leogama Jul 21, 2022
699f30a
sync with branch 'session-excludes'
leogama Jul 22, 2022
04968f3
refonfail: save modules by reference using save_reduce()
leogama Jul 22, 2022
a596126
unpickleable ctypes objects raise ValueError...
leogama Jul 22, 2022
d3837cf
Merge branch 'master' into document-session
leogama Jul 23, 2022
f46d399
move common autodoc options to conf.py and exclude some special members
leogama Jul 24, 2022
bef5795
don't document trace() twice
leogama Jul 24, 2022
953b5e0
fix is_pickled_module()
leogama Jul 26, 2022
e5da1c8
deteail the effects of 'module' argument in load_module() and rename …
leogama Jul 30, 2022
2e4887c
Better describe the side effects and the usage of the returned value …
leogama Jul 30, 2022
be319c8
describe session module behavior and use cases
leogama Jul 30, 2022
a9ea883
add Python License and copyright notice for modified code as specifie…
leogama Aug 1, 2022
b722431
revert addition of PSF license; add link to license
leogama Aug 2, 2022
2a7e984
_open(): cover all the possible file opening modes
leogama Aug 3, 2022
fa4fa85
grammar
leogama Aug 3, 2022
92318a7
better document Pickler.save() and Pickler._save_module_dict()
leogama Aug 3, 2022
4fc2f5f
Merge branch 'master' into document-session
leogama Aug 4, 2022
0e365f5
move session settings to session.py; changes to refonfail
leogama Aug 4, 2022
9c54e34
add _TruncatableWriter to handle 'refonfail' with non-seekable streams
leogama Aug 4, 2022
ffdd180
update 'refonfail' example
leogama Aug 4, 2022
d5b1701
Merge branch 'master' into document-session
leogama Aug 13, 2022
f60d239
merge the two save() methods and save_module_dict() with _save_module…
leogama Aug 14, 2022
4fe577b
minor
leogama Aug 14, 2022
d059d84
grammar; keep __weakref__ attribute in docs
leogama Aug 14, 2022
8bf6157
Add option 'refonfail' for dump_module (enabled by default)
leogama Oct 4, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions dill/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
from .__info__ import __version__, __author__, __doc__, __license__
except: # pragma: no cover
import os
import sys
import sys
parent = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
sys.path.append(parent)
# get distribution meta info
# get distribution meta info
from version import (__version__, __author__,
get_license_text, get_readme_as_rst)
__license__ = get_license_text(os.path.join(parent, 'LICENSE'))
Expand All @@ -24,14 +24,14 @@


from ._dill import (
Pickler, Unpickler,
check, copy, dump, dumps, load, loads, pickle, pickles, register,
DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, CONTENTS_FMODE, FILE_FMODE, HANDLE_FMODE,
dump, dumps, load, loads, copy,
Pickler, Unpickler, register, pickle, pickles, check,
DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, HANDLE_FMODE, CONTENTS_FMODE, FILE_FMODE,
PickleError, PickleWarning, PicklingError, PicklingWarning, UnpicklingError,
UnpicklingWarning,
)
from .session import (
dump_module, load_module, load_module_asdict,
dump_module, load_module, load_module_asdict, is_pickled_module,
dump_session, load_session # backward compatibility
)
from . import detect, logger, session, source, temp
Expand All @@ -42,8 +42,6 @@
# make sure "trace" is turned off
logger.trace(False)

from importlib import reload

objects = {}
# local import of dill._objects
#from . import _objects
Expand All @@ -68,6 +66,7 @@ def load_types(pickleable=True, unpickleable=True):
Returns:
None
"""
from importlib import reload
# local import of dill.objects
from . import _objects
if pickleable:
Expand Down
175 changes: 137 additions & 38 deletions dill/_dill.py

Large diffs are not rendered by default.

24 changes: 13 additions & 11 deletions dill/logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,22 +129,26 @@ def trace_setup(self, pickler):
if not dill._dill.is_dill(pickler, child=False):
return
if self.isEnabledFor(logging.INFO):
pickler._trace_depth = 1
pickler._trace_stack = []
pickler._size_stack = []
else:
pickler._trace_depth = None
def trace(self, pickler, msg, *args, **kwargs):
if not hasattr(pickler, '_trace_depth'):
pickler._trace_stack = None
def trace(self, pickler, msg, *args, obj=None, **kwargs):
if not hasattr(pickler, '_trace_stack'):
logger.info(msg, *args, **kwargs)
return
if pickler._trace_depth is None:
if pickler._trace_stack is None:
return
extra = kwargs.get('extra', {})
pushed_obj = msg.startswith('#')
if not pushed_obj:
if obj is None:
obj = args[-1]
pickler._trace_stack.append(id(obj))
size = None
try:
# Streams are not required to be tellable.
size = pickler._file.tell()
size = pickler._file_tell()
frame = pickler.framer.current_frame
try:
size += frame.tell()
Expand All @@ -159,13 +163,11 @@ def trace(self, pickler, msg, *args, **kwargs):
else:
size -= pickler._size_stack.pop()
extra['size'] = size
if pushed_obj:
pickler._trace_depth -= 1
extra['depth'] = pickler._trace_depth
extra['depth'] = len(pickler._trace_stack)
kwargs['extra'] = extra
self.info(msg, *args, **kwargs)
if not pushed_obj:
pickler._trace_depth += 1
if pushed_obj:
pickler._trace_stack.pop()

class TraceFormatter(logging.Formatter):
"""
Expand Down
Loading