From 2b644228dee21399bfed13561ba812f24dc435ec Mon Sep 17 00:00:00 2001 From: Andrew Tritt Date: Tue, 20 Aug 2019 17:50:53 -0700 Subject: [PATCH 1/3] pickle type map in setup.py --- setup.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2f1d75e4c..ba7023ca1 100755 --- a/setup.py +++ b/setup.py @@ -5,6 +5,20 @@ import versioneer +########## BEGIN: PICKLE TYPE MAP +import sys +import os +import pickle + +sys.path.append('src') +import pynwb +tm = pynwb.get_type_map() + +tm_pkl_path = 'typemap.pkl' +with open(os.path.join('src', 'pynwb', tm_pkl_path) , 'wb') as f: + pickle.dump(tm, f, pickle.HIGHEST_PROTOCOL) +########## END: PICKLE TYPE MAP + with open('README.rst', 'r') as fp: readme = fp.read() @@ -33,7 +47,8 @@ 'install_requires': reqs, 'packages': pkgs, 'package_dir': {'': 'src'}, - 'package_data': {'pynwb': ["%s/*.yaml" % schema_dir, "%s/*.json" % schema_dir]}, + #'package_data': {'pynwb': ["%s/*.yaml" % schema_dir, "%s/*.json" % schema_dir]}, + 'package_data': {'pynwb': [tm_pkl_path]}, 'classifiers': [ "Programming Language :: Python", "Programming Language :: Python :: 3.5", From fefee89bc45c15f9dd715b5b26c4c7639b66c464 Mon Sep 17 00:00:00 2001 From: Andrew Tritt Date: Tue, 20 Aug 2019 17:51:07 -0700 Subject: [PATCH 2/3] load type map from pickle if possible --- src/pynwb/__init__.py | 198 +++++++++++++++++++---------------- src/pynwb/legacy/__init__.py | 14 +-- 2 files changed, 114 insertions(+), 98 deletions(-) diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 7ee71f1b9..e796eb732 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -14,11 +14,62 @@ from hdmf.backends.io import HDMFIO # noqa: E402 from hdmf.backends.hdf5 import HDF5IO as _HDF5IO # noqa: E402 from hdmf.validate import ValidatorMap # noqa: E402 -from hdmf.build import BuildManager # noqa: E402 +from hdmf.build import BuildManager, TypeMap # noqa: E402 from .spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace # noqa: E402 + +__rct_kwargs = list() + + +# a function to register a container classes with the global map +@docval({'name': 'neurodata_type', 'type': str, 'doc': 'the neurodata_type to get the spec for'}, + {'name': 'namespace', 'type': str, 'doc': 'the name of the namespace'}, + {"name": "container_cls", "type": type, + "doc": "the class to map to the specified neurodata_type", 'default': None}, + is_method=False) +def register_class(**kwargs): + """Register an NWBContainer class to use for reading and writing a neurodata_type from a specification + If container_cls is not specified, returns a decorator for registering an NWBContainer subclass + as the class for neurodata_type in namespace. + """ + neurodata_type, namespace, container_cls = getargs('neurodata_type', 'namespace', 'container_cls', kwargs) + + def _dec(cls): + __rct_kwargs.append({'data_type': neurodata_type, 'namespace': namespace, 'container_cls': cls}) + return cls + if container_cls is None: + return _dec + else: + _dec(container_cls) + + +__rm_kwargs = list() + + +# a function to register an object mapper for a container class +@docval({"name": "container_cls", "type": type, + "doc": "the Container class for which the given ObjectMapper class gets used for"}, + {"name": "mapper_cls", "type": type, "doc": "the ObjectMapper class to use to map", 'default': None}, + is_method=False) +def register_map(**kwargs): + """Register an ObjectMapper to use for a Container class type + If mapper_cls is not specified, returns a decorator for registering an ObjectMapper class + as the mapper for container_cls. If mapper_cls specified, register the class as the mapper for container_cls + """ + container_cls, mapper_cls = getargs('container_cls', 'mapper_cls', kwargs) + + def _dec(cls): + __rm_kwargs.append({'mapper_cls': cls, 'container_cls': container_cls}) + return cls + if mapper_cls is None: + return _dec + else: + _dec(mapper_cls) + + __core_ns_file_name = 'nwb.namespace.yaml' +__typemap_pkl_name = 'typemap.pkl' def __get_resources(): @@ -26,6 +77,7 @@ def __get_resources(): from os.path import join ret = dict() ret['namespace_path'] = join(resource_filename(__name__, 'nwb-schema/core'), __core_ns_file_name) + ret['cached_typemap_path'] = resource_filename(__name__, __typemap_pkl_name) return ret @@ -34,15 +86,67 @@ def _get_resources(): return __get_resources() -# a global namespace catalog -global __NS_CATALOG +# a global type map global __TYPE_MAP -__NS_CATALOG = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace) -from hdmf.build import TypeMap as TypeMap # noqa: E402 +@docval({'name': 'namespace_path', 'type': str, + 'doc': 'the path to the YAML with the namespace definition'}, + returns="the namespaces loaded from the given file", rtype=tuple, + is_method=False) +def load_namespaces(**kwargs): + ''' + Load namespaces from file + ''' + namespace_path = getargs('namespace_path', kwargs) + return __TYPE_MAP.load_namespaces(namespace_path) + + +def available_namespaces(): + return __TYPE_MAP.namespace_catalog.namespaces + + +# load the core namespace i.e. base NWB specification +__resources = __get_resources() +if os.path.exists(__resources['cached_typemap_path']): + import pickle + with open(__resources['cached_typemap_path'], 'rb') as f: + __TYPE_MAP = pickle.load(f) +elif os.path.exists(__resources['namespace_path']): + __TYPE_MAP = TypeMap(NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace)) -__TYPE_MAP = TypeMap(__NS_CATALOG) + load_namespaces(__resources['namespace_path']) + + # import these so the TypeMap gets populated + from . import io as __io # noqa: F401,E402 + + from . import core # noqa: F401,E402 + from . import base # noqa: F401,E402 + from . import file # noqa: F401,E402 + from . import behavior # noqa: F401,E402 + from . import device # noqa: F401,E402 + from . import ecephys # noqa: F401,E402 + from . import epoch # noqa: F401,E402 + from . import icephys # noqa: F401,E402 + from . import image # noqa: F401,E402 + from . import misc # noqa: F401,E402 + from . import ogen # noqa: F401,E402 + from . import ophys # noqa: F401,E402 + from . import retinotopy # noqa: F401,E402 + + for _ in __rct_kwargs: + __TYPE_MAP.register_container_type(**_) + for _ in __rm_kwargs: + __TYPE_MAP.register_map(**_) +else: + raise RuntimeError("Unable to load a TypeMap") + + +NWBContainer = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'NWBContainer') +NWBData = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'NWBData') +TimeSeries = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'TimeSeries') +ProcessingModule = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'ProcessingModule') +NWBFile = __TYPE_MAP.get_container_cls(CORE_NAMESPACE, 'NWBFile') @docval({'name': 'extensions', 'type': (str, TypeMap, list), @@ -94,71 +198,6 @@ def get_manager(**kwargs): return BuildManager(type_map) -@docval({'name': 'namespace_path', 'type': str, - 'doc': 'the path to the YAML with the namespace definition'}, - returns="the namespaces loaded from the given file", rtype=tuple, - is_method=False) -def load_namespaces(**kwargs): - ''' - Load namespaces from file - ''' - namespace_path = getargs('namespace_path', kwargs) - return __TYPE_MAP.load_namespaces(namespace_path) - - -# load the core namespace i.e. base NWB specification -__resources = __get_resources() -if os.path.exists(__resources['namespace_path']): - load_namespaces(__resources['namespace_path']) - - -def available_namespaces(): - return __NS_CATALOG.namespaces - - -# a function to register a container classes with the global map -@docval({'name': 'neurodata_type', 'type': str, 'doc': 'the neurodata_type to get the spec for'}, - {'name': 'namespace', 'type': str, 'doc': 'the name of the namespace'}, - {"name": "container_cls", "type": type, - "doc": "the class to map to the specified neurodata_type", 'default': None}, - is_method=False) -def register_class(**kwargs): - """Register an NWBContainer class to use for reading and writing a neurodata_type from a specification - If container_cls is not specified, returns a decorator for registering an NWBContainer subclass - as the class for neurodata_type in namespace. - """ - neurodata_type, namespace, container_cls = getargs('neurodata_type', 'namespace', 'container_cls', kwargs) - - def _dec(cls): - __TYPE_MAP.register_container_type(namespace, neurodata_type, cls) - return cls - if container_cls is None: - return _dec - else: - _dec(container_cls) - - -# a function to register an object mapper for a container class -@docval({"name": "container_cls", "type": type, - "doc": "the Container class for which the given ObjectMapper class gets used for"}, - {"name": "mapper_cls", "type": type, "doc": "the ObjectMapper class to use to map", 'default': None}, - is_method=False) -def register_map(**kwargs): - """Register an ObjectMapper to use for a Container class type - If mapper_cls is not specified, returns a decorator for registering an ObjectMapper class - as the mapper for container_cls. If mapper_cls specified, register the class as the mapper for container_cls - """ - container_cls, mapper_cls = getargs('container_cls', 'mapper_cls', kwargs) - - def _dec(cls): - __TYPE_MAP.register_map(container_cls, cls) - return cls - if mapper_cls is None: - return _dec - else: - _dec(mapper_cls) - - # a function to get the container class for a give type @docval({'name': 'neurodata_type', 'type': str, 'doc': 'the neurodata_type to get the NWBConatiner class for'}, @@ -232,23 +271,6 @@ def __init__(self, **kwargs): super(NWBHDF5IO, self).__init__(path, manager=manager, mode=mode, file=file_obj, comm=comm) -from . import io as __io # noqa: F401,E402 -from .core import NWBContainer, NWBData # noqa: F401,E402 -from .base import TimeSeries, ProcessingModule # noqa: F401,E402 -from .file import NWBFile # noqa: E402, F401 - -from . import behavior # noqa: F401,E402 -from . import device # noqa: F401,E402 -from . import ecephys # noqa: F401,E402 -from . import epoch # noqa: F401,E402 -from . import icephys # noqa: F401,E402 -from . import image # noqa: F401,E402 -from . import misc # noqa: F401,E402 -from . import ogen # noqa: F401,E402 -from . import ophys # noqa: F401,E402 -from . import retinotopy # noqa: F401,E402 -from . import legacy # noqa: F401,E402 - from ._version import get_versions # noqa: E402 __version__ = get_versions()['version'] del get_versions diff --git a/src/pynwb/legacy/__init__.py b/src/pynwb/legacy/__init__.py index 4c03eef79..e9321c194 100644 --- a/src/pynwb/legacy/__init__.py +++ b/src/pynwb/legacy/__init__.py @@ -1,21 +1,15 @@ -from hdmf.spec import NamespaceCatalog from hdmf.utils import docval, getargs -from ..spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace -from .. import _get_resources, get_type_map, NWBContainer +from .. import get_type_map, NWBContainer from .map import ObjectMapperLegacy as ObjectMapper from .map import TypeMapLegacy as TypeMap -__NS_CATALOG = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace) -__TYPE_MAP = TypeMap(__NS_CATALOG) - # load the core namespace i.e. base NWB specification -__resources = _get_resources() -__TYPE_MAP.load_namespaces(__resources['namespace_path']) -__TYPE_MAP.merge(get_type_map()) - +dflt_tm = get_type_map() +__TYPE_MAP = TypeMap(dflt_tm.namespace_catalog) +__TYPE_MAP.merge(dflt_tm) # Register new ObjectMapper with the new TypeMap: __TYPE_MAP.register_map(NWBContainer, ObjectMapper) From 973962255cd1495c3046ceb9442f8a78fb682b41 Mon Sep 17 00:00:00 2001 From: Andrew Tritt Date: Wed, 21 Aug 2019 09:50:31 -0700 Subject: [PATCH 3/3] add custom build_py command --- setup.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/setup.py b/setup.py index ba7023ca1..e210c0434 100755 --- a/setup.py +++ b/setup.py @@ -1,23 +1,29 @@ # -*- coding: utf-8 -*- from setuptools import setup, find_packages +from setuptools.command.build_py import build_py import re import versioneer -########## BEGIN: PICKLE TYPE MAP -import sys -import os -import pickle +class BuildPyCommand(build_py): + tm_pkl_path = 'typemap.pkl' + def run(self): + ########## BEGIN: PICKLE TYPE MAP + import sys + import os + import pickle -sys.path.append('src') -import pynwb -tm = pynwb.get_type_map() + sys.path.append('src') + import pynwb + tm = pynwb.get_type_map() -tm_pkl_path = 'typemap.pkl' -with open(os.path.join('src', 'pynwb', tm_pkl_path) , 'wb') as f: - pickle.dump(tm, f, pickle.HIGHEST_PROTOCOL) -########## END: PICKLE TYPE MAP + pkl_path = os.path.join('src', 'pynwb', self.tm_pkl_path) + sys.stdout.write('pickling type map to %s\n' % pkl_path) + with open(pkl_path, 'wb') as f: + pickle.dump(tm, f, pickle.HIGHEST_PROTOCOL) + ########## END: PICKLE TYPE MAP + super().run() with open('README.rst', 'r') as fp: readme = fp.read() @@ -48,7 +54,7 @@ 'packages': pkgs, 'package_dir': {'': 'src'}, #'package_data': {'pynwb': ["%s/*.yaml" % schema_dir, "%s/*.json" % schema_dir]}, - 'package_data': {'pynwb': [tm_pkl_path]}, + 'package_data': {'pynwb': [BuildPyCommand.tm_pkl_path]}, 'classifiers': [ "Programming Language :: Python", "Programming Language :: Python :: 3.5", @@ -77,7 +83,8 @@ 'NWB ' 'NWB:N ' 'NeurodataWithoutBorders', - 'zip_safe': False + 'zip_safe': False, + 'cmdclass': {'build_py': BuildPyCommand} } if __name__ == '__main__':