Merge pull request #204 from nikulukani/develop

Binary Deserialization support though c binding

.travis.yml

@@ -6,6 +6,7 @@ python:
 
 install:
 - pip install coveralls
+- pip install pyorient_native
 - "./ci/start-ci.sh $ORIENTDB_VERSION"
 
 cache:

pyorient/messages/base.py

@@ -67,7 +67,11 @@ def get_serializer(self):
         Lazy return of the serialization, we retrive the type from the :class: `OrientSocket <pyorient.orient.OrientSocket>` object
         :return: an Instance of the serializer suitable for decoding or encoding
         """
-        return OrientSerialization.get_impl(self._orientSocket.serialization_type)
+        if self._orientSocket.serialization_type==OrientSerialization.Binary:
+            return OrientSerialization.get_impl(self._orientSocket.serialization_type,
+                                                self._orientSocket._props)
+        else:
+            return OrientSerialization.get_impl(self._orientSocket.serialization_type)
 
     def get_orient_socket_instance(self):
         return self._orientSocket

pyorient/messages/connection.py

@@ -6,7 +6,7 @@
     FIELD_STRINGS, FIELD_BOOLEAN, FIELD_STRING, NAME, SUPPORTED_PROTOCOL, \
     VERSION, SHUTDOWN_OP
 from ..utils import need_connected
-from ..serializations import OrientSerialization
+#from ..serializations import OrientSerialization
 
 #
 # Connect
@@ -19,7 +19,6 @@ def __init__(self, _orient_socket):
         self._user = ''
         self._pass = ''
         self._client_id = ''
-        self._serialization_type = OrientSerialization.CSV
         self._need_token = False
         self._append( ( FIELD_BYTE, CONNECT_OP ) )
 
@@ -40,11 +39,8 @@ def prepare(self, params=None ):
 
         self._append( ( FIELD_STRING, self._client_id ) )
 
-        # Set the serialization type on the shared socket object
-        self._orientSocket.serialization_type = self._serialization_type
-
         if self.get_protocol() > 21:
-            self._append( ( FIELD_STRING, self._serialization_type ) )
+            self._append( ( FIELD_STRING, self._orientSocket.serialization_type ) )
             if self.get_protocol() > 26:
                 self._append( ( FIELD_BOOLEAN, self._request_token ) )
                 if self.get_protocol() > 32:
@@ -125,4 +121,4 @@ def set_user(self, _user):
 
     def set_pass(self, _pass):
         self._pass = _pass
-        return self
+        return self

pyorient/messages/database.py

@@ -51,7 +51,6 @@ def __init__(self, _orient_socket):
         self._client_id = ''
         self._db_name = ''
         self._db_type = DB_TYPE_DOCUMENT
-        self._serialization_type = OrientSerialization.CSV
         self._append(( FIELD_BYTE, DB_OPEN_OP ))
         self._need_token = False
 
@@ -64,8 +63,7 @@ def prepare(self, params=None):
                 self._pass = params[2]
                 self.set_db_type(params[3])
                 self._client_id = params[4]
-                self._serialization_type = params[5]
-
+
             except IndexError:
                 # Use default for non existent indexes
                 pass
@@ -74,11 +72,9 @@ def prepare(self, params=None):
         self._append(( FIELD_SHORT, SUPPORTED_PROTOCOL ))
         self._append(( FIELD_STRING, self._client_id ))
 
-        # Set the serialization type on the shared socket object
-        self._orientSocket.serialization_type = self._serialization_type
 
         if self.get_protocol() > 21:
-            self._append(( FIELD_STRING, self._serialization_type ))
+            self._append(( FIELD_STRING, self._orientSocket.serialization_type ))
             if self.get_protocol() > 26:
                 self._append(( FIELD_BOOLEAN, self._request_token ))
                 if self.get_protocol() >= 36:
@@ -152,9 +148,6 @@ def fetch_response(self):
         # set database opened
         self._orientSocket.db_opened = self._db_name
 
-        # set serialization type, as global in the orient socket class
-        self._orientSocket.serialization_type = self._serialization_type
-
         return info, clusters, self._node_list
         # self._cluster_map = self._orientSocket.cluster_map = \
         #     Information([clusters, response, self._orientSocket])

pyorient/ogm/config.py

@@ -1,6 +1,8 @@
 import os
 import re
 
+from pyorient.serializations import OrientSerialization
+
 try:
     from urllib.parse import urlparse, urlunparse
 except ImportError:
@@ -9,7 +11,7 @@
 class Config(object):
     """Specifies how to connect to OrientDB server."""
     def __init__(self, host, port, user, cred, db_name=None, storage='memory'
-                 , initial_drop=False):
+                 , initial_drop=False, serialization_type=OrientSerialization.CSV):
         """
         :param initial_drop: Useful for testing; signal that any existing
         database with this configuration should be dropped on connect.
@@ -21,11 +23,11 @@ def __init__(self, host, port, user, cred, db_name=None, storage='memory'
         self.db_name = db_name
         self.storage = storage
         self.initial_drop = initial_drop
-
+        self.serialization_type = serialization_type
         self.scripts = None
 
     @classmethod
-    def from_url(cls, url, user, cred, initial_drop=False):
+    def from_url(cls, url, user, cred, initial_drop=False, serialization_type=OrientSerialization.CSV):
         url_exp = re.compile(r'^(\w+:\/\/)?(.*)')
         url_match = url_exp.match(url)
         if not url_match.group(1):
@@ -39,11 +41,11 @@ def from_url(cls, url, user, cred, initial_drop=False):
         if url_parts.path:
             db_name = os.path.basename(url_parts.path)
             return cls(url_parts.hostname, url_parts.port, user, cred, db_name
-                       , url_parts.scheme, initial_drop)
+                       , url_parts.scheme, initial_drop, serialization_type)
         else:
             db_name = url_parts.netloc
             return cls(None, url_parts.port, user, cred, db_name
-                       , url_parts.scheme, initial_drop)
+                       , url_parts.scheme, initial_drop, serialization_type)
 
     def set_database(self, db_name, storage):
         self.db_name = db_name

pyorient/ogm/graph.py

@@ -27,7 +27,7 @@ def __init__(self, config, user=None, cred=None, strict=False):
         :note: user only meaningful when cred also provided.
         """
 
-        self.client = pyorient.OrientDB(config.host, config.port)
+        self.client = pyorient.OrientDB(config.host, config.port, config.serialization_type)
         self.client.connect(config.user, config.cred)
 
         self.config = config

pyorient/orient.py

@@ -23,6 +23,31 @@
 
 from .utils import dlog
 
+type_map = {'BOOLEAN' :0,
+            'INTEGER' :1,
+            'SHORT' :2,
+            'LONG' :3,
+            'FLOAT' :4,
+            'DOUBLE' :5,
+            'DATETIME':6,
+            'STRING':7,
+            'BINARY':8,
+            'EMBEDDED':9,
+            'EMBEDDEDLIST':10,
+            'EMBEDDEDSET':11,
+            'EMBEDDEDMAP':12,
+            'LINK' :13,
+            'LINKLIST':14,
+            'LINKSET' :15,
+            'LINKMAP' :16,
+            'BYTE' : 17,
+            'TRANSIENT' : 18,
+            'DATE' :19,
+            'CUSTOM' : 20,
+            'DECIMAL' : 21,
+            'LINKBAG' : 22,
+            'ANY' : 23}
+
 class OrientSocket(object):
     '''Class representing the binary connection to the database, it does all the low level comunication
     And holds information on server version and cluster map
@@ -34,7 +59,7 @@ class OrientSocket(object):
     :param port: integer port of the server
 
     '''
-    def __init__(self, host, port):
+    def __init__(self, host, port, serialization_type=OrientSerialization.CSV ):
 
         self.connected = False
         self.host = host
@@ -44,8 +69,9 @@ def __init__(self, host, port):
         self.session_id = -1
         self.auth_token = b''
         self.db_opened = None
-        self.serialization_type = OrientSerialization.CSV
+        self.serialization_type = serialization_type
         self.in_transaction = False
+        self._props = None
 
     def get_connection(self):
         if not self.connected:
@@ -207,9 +233,9 @@ class OrientDB(object):
         TxCommitMessage="pyorient.messages.commands",
     )
 
-    def __init__(self, host='localhost', port=2424):
+    def __init__(self, host='localhost', port=2424, serialization_type=OrientSerialization.CSV):
         if not isinstance(host, OrientSocket):
-            connection = OrientSocket(host, port)
+            connection = OrientSocket(host, port, serialization_type)
         else:
             connection = host
 
@@ -226,7 +252,8 @@ def __init__(self, host='localhost', port=2424):
         self._cluster_map = None
         self._cluster_reverse_map = None
         self._connection = connection
-
+        self._serialization_type = serialization_type
+
     def __getattr__(self, item):
 
         # No special handling for private attributes/methods.
@@ -278,13 +305,13 @@ def get_session_token( self ):
 
     # SERVER COMMANDS
 
-    def connect(self, user, password, client_id='', serialization_type=OrientSerialization.CSV):
+    def connect(self, user, password, client_id=''):
         '''Connect to the server without opening any database
 
         :param user: the username of the user on the server. Example: "root"
         :param password: the password of the user on the server. Example: "37aed6392"
         :param client_id: client's id - can be null for clients. In clustered configurations it's the distributed node ID as TCP host:port
-        :param serialization_type: the serialization format required by the client, now it can be just OrientSerialization.CSV
+        ## :param serialization_type: the serialization format required by the client, now it can be just OrientSerialization.CSV
 
         Usage to open a connection as root::
 
@@ -294,7 +321,7 @@ def connect(self, user, password, client_id='', serialization_type=OrientSeriali
 
         '''
         return self.get_message("ConnectMessage") \
-            .prepare((user, password, client_id, serialization_type)).send().fetch_response()
+            .prepare((user, password, client_id, self._serialization_type)).send().fetch_response()
 
     def db_count_records(self):
         '''Returns the number of records in the currently open database.
@@ -387,6 +414,10 @@ def db_open(self, db_name, user, password, db_type=DB_TYPE_DOCUMENT, client_id='
         self._reload_clusters()
         self.nodes = nodes
 
+        # store property id->property name, type map for binary serialization
+
+        self.update_properties()
+
         return self.clusters
 
     def db_reload(self):
@@ -398,8 +429,22 @@ def db_reload(self):
         self.clusters = self.get_message("DbReloadMessage") \
             .prepare([]).send().fetch_response()
         self._reload_clusters()
+        self.update_properties()
         return self.clusters
 
+
+    def update_properties(self):
+        '''
+        This method fetches the global Properties from the server. The properties are used
+        for deserializing based on propery index if using binary serialization. This method
+        should be called after any manual command that may result in modifications to the
+        properties table, for example, "Create property ... " or "Create class .." followed
+        by "Create vertex set ... "
+        '''
+        if self._serialization_type==OrientSerialization.Binary:
+            self._connection._props = {x['id']:[x['name'], type_map[x['type']]] for x in
+                        self.command("select from #0:1")[0].oRecordData['globalProperties']}
+
     def shutdown(self, *args):
         return self.get_message("ShutdownMessage") \
             .prepare(args).send().fetch_response()

pyorient/serializations.py

@@ -4,21 +4,48 @@
 from decimal import Decimal
 from .otypes import OrientRecordLink, OrientRecord, OrientBinaryObject
 from .exceptions import PyOrientBadMethodCallException
-
+try:
+    import pyorient_native
+    binary_support=True
+except:
+    binary_support=False
 
 class OrientSerializationBinary(object):
-    def __init__(self):
+    def __init__(self, props):
         self.className = None
         self.data = {}
         self.type = OrientSerialization.Binary
-
+        self.props = props
+        self._writer = None
+
     def decode(self, content):
-        raise NotImplementedError
+        if not binary_support:
+            raise Exception("To support Binary Serialization,\
+                            pyorient_native must be installed")
+        clsname, data = pyorient_native.deserialize(content,
+                                    content.__sizeof__(), self.props)
+        rels = [k for k in data.keys() if ('in_' in k or 'out_' in k
+                                       or k=='in' or k=='out')] 
+        for k in rels:
+            if isinstance(data[k],list):
+                for i in range(len(data[k])):
+                    data[k][i] = OrientRecordLink(str(data[k][i][1]) + ':' +
+                                                  str(data[k][i][2]))
+            elif isinstance(data[k],tuple):
+                data[k] = OrientRecordLink(str(data[k][1]) + ':' +
+                                                  str(data[k][2]))
+        return [clsname, data]
 
     def encode(self, record):
-        raise NotImplementedError
-
-
+        if not binary_support:
+            raise Exception("To support Binary Serialization,\
+                            pyorient_native must be installed")
+        if record:
+            return pyorient_native.serialize(record)
+        else:
+            return None
+
+
 class OrientSerializationCSV(object):
     def __init__(self):
         self.className = None
@@ -557,7 +584,7 @@ class OrientSerialization(object):
     Binary = "ORecordSerializerBinary"
 
     @classmethod
-    def get_impl(cls, impl):
+    def get_impl(cls, impl, props=None):
         impl_map = {
             cls.CSV: OrientSerializationCSV,
             cls.Binary: OrientSerializationBinary,
@@ -567,4 +594,7 @@ def get_impl(cls, impl):
             raise PyOrientBadMethodCallException(
                 impl + ' is not an availableserialization type', []
             )
-        return implementation()
+        if impl == cls.Binary:
+            return implementation(props)
+        else:
+            return implementation()