Merge pull request #100 from neuro-ml/dev

Basic support of pickling

Author: STNLd2

connectome/__version__.py

@@ -1 +1 @@
-__version__ = '0.8.1'
+__version__ = '0.9.0'

connectome/containers/base.py

@@ -1,5 +1,4 @@
 import logging
-import warnings
 from operator import itemgetter
 from typing import Callable, Optional, Union
 
@@ -11,44 +10,15 @@
 from ..utils import NameSet, StringsLike, check_for_duplicates, node_to_dict
 from .context import BagContext, ChainContext, Context, NoContext, update_map
 
-__all__ = 'Container', 'EdgesBag'
+__all__ = 'EdgesBag',
 
 logger = logging.getLogger(__name__)
 
 
-class Container:
-    def __init__(self):
-        warnings.warn(
-            'The container interface is deprecated and will be merged with `EdgesBag` soon',
-            UserWarning, stacklevel=2
-        )
-        warnings.warn(
-            'The container interface is deprecated and will be merged with `EdgesBag` soon',
-            DeprecationWarning, stacklevel=2
-        )
-
-    def wrap(self, container: 'EdgesBag') -> 'EdgesBag':
-        raise NotImplementedError
-
-
 class EdgesBag:
     def __init__(self, inputs: Nodes, outputs: Nodes, edges: BoundEdges, context: Optional[Context], *,
-                 virtual_nodes: Optional[NameSet] = None, virtual: Optional[NameSet] = None,
-                 persistent_nodes: Optional[NameSet] = None, persistent: Optional[NameSet] = None,
-                 optional_nodes: Optional[NodeSet] = None, optional: Optional[NodeSet] = None):
-        if virtual_nodes is not None:
-            assert virtual is None
-            warnings.warn('The "virtual_nodes" argument is deprecated. Use `virtual` instead', stacklevel=2)
-            virtual = virtual_nodes
-        if optional_nodes is not None:
-            assert optional is None
-            warnings.warn('The "optional_nodes" argument is deprecated. Use `optional` instead', stacklevel=2)
-            optional = optional_nodes
-        if persistent_nodes is not None:
-            assert persistent is None
-            warnings.warn('The "persistent_nodes" argument is deprecated. Use `persistent` instead', stacklevel=2)
-            persistent = persistent_nodes
-
+                 virtual: Optional[NameSet] = None, persistent: Optional[NameSet] = None,
+                 optional: Optional[NodeSet] = None):
         if virtual is None:
             virtual = set()
         if persistent is None:
@@ -103,22 +73,6 @@ def loopback(self, func: Callable, inputs: StringsLike, output: StringsLike) ->
             virtual=None, persistent=None, optional=state.optional | new_optionals,
         )
 
-    # TODO: deprecated
-    @property
-    def persistent_nodes(self):
-        warnings.warn('This attribute is deprecated. Use `persistent` instead', stacklevel=2)
-        return self.persistent
-
-    @property
-    def optional_nodes(self):
-        warnings.warn('This attribute is deprecated. Use `optional` instead', stacklevel=2)
-        return self.optional
-
-    @property
-    def virtual_nodes(self):
-        warnings.warn('This attribute is deprecated. Use `virtual` instead', stacklevel=2)
-        return self.virtual
-
 
 def normalize_bag(inputs: Nodes, outputs: Nodes, edges: BoundEdges, virtuals: NameSet, optionals: NodeSet,
                   persistent_nodes: NameSet, allow_missing_inputs: bool = True):

connectome/engine/__init__.py

@@ -2,4 +2,4 @@
 from .compiler import GraphCompiler
 from .edges import *
 from .graph import Graph
-from .node_hash import ApplyHash, CustomHash, FilterHash, GraphHash, LeafHash, NodeHash, NodeHashes, TupleHash
+from .node_hash import ApplyHash, CustomHash, GraphHash, LeafHash, NodeHash, NodeHashes

connectome/engine/graph.py

@@ -22,8 +22,6 @@ def __init__(self, inputs: TreeNodes, output: TreeNode):
         self.output = output
         self.counts = counts
         self.__signature__ = signature
-        # TODO: deprecate
-        self.call = self.__call__
 
     def __call__(*args, **kwargs):
         self, *args = args

connectome/engine/node_hash.py

@@ -1,4 +1,3 @@
-import warnings
 from typing import Any, Callable, Sequence, Tuple
 
 NODE_TYPES = set()
@@ -75,46 +74,3 @@ def __init__(self, marker: Any, *children: NodeHash):
             (self.type, marker, *(h.value for h in children)),
             (self.type, marker, *children),
         )
-
-
-# TODO: deprecated
-class CompoundBase(NodeHash):
-    type = None
-
-    def __init__(self, *children: NodeHash):
-        warnings.warn('This interface is deprecated', DeprecationWarning)
-        warnings.warn('This interface is deprecated', UserWarning)
-        super().__init__(
-            (self.type, *(h.value for h in children)),
-            (self.type, *children),
-        )
-
-
-class TupleHash(ApplyHash):
-    type = -1
-
-    def __init__(self, *children: NodeHash):
-        warnings.warn('This interface is deprecated. Use ApplyHash instead', DeprecationWarning)
-        warnings.warn('This interface is deprecated. Use ApplyHash instead', UserWarning)
-        super().__init__(tuple, *children)
-
-
-class FilterHash(ApplyHash):
-    type = -2
-
-    def __init__(self, graph: GraphHash, values: NodeHash):
-        warnings.warn('This interface is deprecated. Use ApplyHash instead', DeprecationWarning)
-        warnings.warn('This interface is deprecated. Use ApplyHash instead', UserWarning)
-        super().__init__(filter, graph, values)
-
-
-class MergeHash(CustomHash):
-    type = -3
-
-    def __init__(self, *children: NodeHash):
-        warnings.warn('This interface is deprecated. Use CustomHash instead', DeprecationWarning)
-        warnings.warn('This interface is deprecated. Use CustomHash instead', UserWarning)
-        super().__init__('connectome.SwitchEdge', *children)
-
-
-PrecomputeHash = NodeHash

connectome/interface/factory.py

@@ -58,13 +58,6 @@ def is_detectable(value):
 BUILTIN_DECORATORS = staticmethod, classmethod, property
 
 
-class FactoryLayer(CallableLayer):
-    def __init__(self, container: EdgesBag, properties: Iterable[str], special_methods: Iterable[str]):
-        warnings.warn('This class is deprecated', DeprecationWarning)
-        warnings.warn('This class is deprecated', UserWarning)
-        super().__init__(container, properties)
-
-
 class GraphFactory:
     layer_cls: Type[Layer] = CallableLayer
 

connectome/interface/metaclasses.py

@@ -3,6 +3,7 @@
 
 from ..layers import CallableLayer, Layer
 from ..utils import MultiDict
+from .decorators import RuntimeAnnotation
 from .factory import GraphFactory, SourceFactory, TransformFactory, add_from_mixins, add_quals, items_to_container
 
 logger = logging.getLogger(__name__)
@@ -14,6 +15,16 @@ class APIMeta(type):
     def __prepare__(mcs, *args, **kwargs):
         return MultiDict()
 
+    def __getattr__(self, item):
+        # we need this behaviour mostly to support pickling of functions defined inside the class
+        try:
+            value = self.__original__scope__[item]
+            while isinstance(value, RuntimeAnnotation):
+                value = value.__func__
+            return value
+        except KeyError:
+            raise AttributeError(item) from None
+
     def __new__(mcs, class_name, bases, namespace, **flags):
         if '__factory' in flags:
             factory = flags.pop('__factory')
@@ -46,6 +57,8 @@ def __new__(mcs, class_name, bases, namespace, **flags):
             add_from_mixins(namespace, bases)
             scope = factory.make_scope(class_name, namespace)
 
+        # TODO: need a standardized set of magic fields
+        scope['__original__scope__'] = namespace
         return super().__new__(mcs, class_name, (main,), scope, **flags)
 
 

tests/test_interface/test_metaclasses.py

@@ -1,6 +1,8 @@
+import pickle
+
 import pytest
 
-from connectome import Mixin, Source, Transform
+from connectome import Mixin, Source, Transform, meta
 
 
 def test_subclasses():
@@ -33,3 +35,40 @@ class G(C):
             pass
 
     assert str(B()) == 'B()'
+
+
+class A(Transform):
+    def x(x):
+        return x
+
+    def _t(x):
+        return x ** 2
+
+    def y(x, _t):
+        return x + _t
+
+
+class B(Transform):
+    @meta
+    def ids():
+        return '123'
+
+    def x(x):
+        return x
+
+    def _t(x):
+        return x ** 2
+
+    def y(x, _t):
+        return x + _t
+
+
+def test_pickleable():
+    a = A()
+    b = B()
+    assert a.x != A.x
+
+    for f in a.x, a.y, a._compile(dir(a)), b._compile(dir(b)):
+        pickled = pickle.dumps(f)
+        g = pickle.loads(pickled)
+        assert f(0) == g(0)