Skip to content

Commit

Permalink
Added context notion and used it in pickled_callable_wrapper.py
Browse files Browse the repository at this point in the history
  • Loading branch information
ryanfeather committed Nov 30, 2014
1 parent 12fe20e commit 642d9bb
Show file tree
Hide file tree
Showing 6 changed files with 57 additions and 21 deletions.
4 changes: 4 additions & 0 deletions src/doc/API.rst
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ Functions
.. autofunction:: callable_wrapper
.. autofunction:: obfuscator
.. autofunction:: store
.. autofunction:: parsimony_directory
.. autofunction:: context_name
.. autofunction:: set_context




Expand Down
21 changes: 11 additions & 10 deletions src/doc/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ As a toy example, let's do some task that takes a few seconds twice in one sessi

#nothing cached
t0 = time.time()
x = parsimony.generators.PickledCallableWrapper('.parsimony/wrapped_results','x',pd.read_csv,filepath_or_buffer=path)
x = parsimony.generators.PickledCallableWrapper('x',pd.read_csv,filepath_or_buffer=path)
t1 = time.time()
print('Time taken {0:1.5f}s'.format(t1-t0))#nothing happened, default lazy init

Expand All @@ -36,7 +36,7 @@ As a toy example, let's do some task that takes a few seconds twice in one sessi
print("y={0:}".format(y))

t0 = time.time()
x = parsimony.generators.PickledCallableWrapper('.parsimony/wrapped_results','x',pd.read_csv,filepath_or_buffer=path)
x = parsimony.generators.PickledCallableWrapper('x',pd.read_csv,filepath_or_buffer=path)
t1 = time.time()
print('Time taken {0:1.5f}s'.format(t1-t0))#nothing happened, default lazy init
t0 = time.time()
Expand All @@ -48,22 +48,23 @@ As a toy example, let's do some task that takes a few seconds twice in one sessi
Prints
::

Time taken 0.00051s
Time taken 4.82472s
Time taken 0.00023s
Time taken 8.76389s
y=12474872316
Time taken 0.05171s
Time taken 0.00056s
Time taken 0.05012s
Time taken 0.02810s
y=12474872316



However, this is not that useful of an example. Let's run this script again immediately.
::

Time taken 0.00020s
Time taken 0.00087s
Time taken 0.00011s
Time taken 0.00061s
y=12474872316
Time taken 0.00008s
Time taken 0.00063s
Time taken 0.00016s
Time taken 0.00060s
y=12474872316

Because intermediate results were cached, the large CSV file does not need to be read. In fact, the only computation to
Expand Down
2 changes: 1 addition & 1 deletion src/parsimony/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
__version__ = '0.1.2'
__version__ = '0.2.0'
from .generate import generate
from .exceptions import ParsimonyException

Expand Down
2 changes: 1 addition & 1 deletion src/parsimony/configuration/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
from .configuration import store, obfuscator, callable_wrapper
from .configuration import store, obfuscator, callable_wrapper, context_name, set_context, parsimony_directory
34 changes: 31 additions & 3 deletions src/parsimony/configuration/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,40 @@ def obfuscator():
return __obfuscator


def parsimony_directory():
"""Return the directory to store parsimony data in.
:return: .parsimony
"""
directory_path = '.parsimony'
if not os.path.exists(directory_path):
os.makedirs(directory_path)

return directory_path

__context_name = 'Default'


def context_name():
"""Get the context name
:return: Current context name
"""
global __context_name
return __context_name


def set_context(new_context_name):
"""Set the context name. This should be something that can be a directory name.
"""
global __context_name
__context_name = new_context_name


def callable_wrapper(key, function, **parameters):
"""Gets the configured call wrapper. Currently, a default of PickledCallableWrapper is chosen.
:return callable_wrapper: CallableWrapper Generator object
"""
if not os.path.exists('.parsimony/wrapped_results'):
os.makedirs('.parsimony/wrapped_results')

return parsimony.generators.PickledCallableWrapper('.parsimony/wrapped_results', key, function, **parameters)
return parsimony.generators.PickledCallableWrapper(key, function, **parameters)
15 changes: 9 additions & 6 deletions src/parsimony/generators/pickled_callable_wrapper.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
from parsimony.generators import Generator
from parsimony.configuration import context_name, parsimony_directory

import pickle
import os
import string
Expand All @@ -8,19 +10,20 @@ class PickledCallableWrapper(Generator):
""" Generator for callables using pickle as the underlying storage mechanism
"""
def __init__(self, directory, key, function, **parameters):
"""The PickledCallableWrapper is a simple way to cache arbitrary function results.
def __init__(self, key, function, **parameters):
"""The PickledCallableWrapper is a simple way to cache arbitrary function results. Results are stored in
the context subdirectory of the parsimony directory.
:param directory: directory to cache in
:param key: generator key string
:param function: callable handle
:param parameters: key-value parameters for the callable function
"""
self._function = function # keep separate references to avoid copying later
self._param_keys = list(parameters.keys())
if not os.path.exists(directory):
os.makedirs(directory)
self._store_location = self._generate_store_location(directory, key)
self._directory = os.path.join(parsimony_directory(), context_name())
if not os.path.exists(self._directory):
os.makedirs(self._directory)
self._store_location = self._generate_store_location(self._directory, key)
super(PickledCallableWrapper, self).__init__(key, function=function, **parameters)

def up_to_date(self):
Expand Down

0 comments on commit 642d9bb

Please sign in to comment.