Registration POST API and Synch_all implementation framework

dolphin/api/v1/router.py

@@ -37,6 +37,9 @@ def _setup_routes(self, mapper):
                         controller=self.resources['storages'],
                         collection={'sync_all': 'POST'},
                         member={'sync': 'POST'})
+        mapper.resource("storage", "storages",
+                        controller=self.resources["storages"],
+                        member={"action": "POST"})
 
         self.resources['pools'] = pools.create_resource()
         mapper.resource("pool", "pools",

dolphin/api/v1/storages.py

@@ -11,23 +11,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from os_service_types import exc
+from dolphin import context
 from six.moves import http_client
 import webob
 
-from dolphin.api.common import wsgi
+from dolphin.api.common import wsgi, LOG
+from dolphin.db.sqlalchemy import api as db
+from dolphin.resource_manager import storages
+from dolphin.task_manager import manager as taskmanager
 
 
 class StorageController(wsgi.Controller):
 
     def index(self, req):
-        return dict(name="Storage 1")
+        return storages.get_all()
 
     def show(self, req, id):
         return dict(name="Storage 2")
 
     def create(self, req, body):
-        return dict(name="Storage 3")
+
+        return storages.register(self, req, body)
 
     def update(self, req, id, body):
         return dict(name="Storage 4")
@@ -36,13 +41,13 @@ def delete(self, req, id):
         return webob.Response(status_int=http_client.ACCEPTED)
 
     def sync_all(self, req):
-        return dict(name="Sync all storages")
+        taskmgr = taskmanager.TaskManager()
+        ctxt = context.RequestContext()
+        return taskmgr.storage_device_details(ctxt, req)
 
     def sync(self, req, id):
         return dict(name="Sync storage 1")
 
 
 def create_resource():
     return wsgi.Resource(StorageController())
-
-

dolphin/context.py

@@ -33,7 +33,7 @@ class RequestContext(context.RequestContext):
 
     """
 
-    def __init__(self, user_id, project_id, is_admin=None, read_deleted="no",
+    def __init__(self, user_id=None, project_id=None, is_admin=None, read_deleted="no",
                  roles=None, remote_address=None, timestamp=None,
                  request_id=None, auth_token=None, overwrite=True,
                  quota_class=None, service_catalog=None, **kwargs):

dolphin/driver_manager/manager.py

@@ -0,0 +1,73 @@
+# Copyright 2020 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This is sample driver manager code just to run the framework. Need not review"""
+
+import shlex
+import subprocess
+from oslo_log import log
+from oslo_service import time
+from oslo_utils import uuidutils
+
+LOG = log.getLogger(__name__)
+
+DEVICE_INFO = {
+    'id': uuidutils.generate_uuid(),
+    'name': 'string',
+    'description': 'string',
+    'vendor': 'string',
+    'status': 'available',
+    'total_capacity': 'double',
+    'used_capacity': 'double',
+    'free_capacity': 'double',
+    'manufacturer': 'string',
+    'model': 'string',
+    'firmwareVersion': 'string',
+    'serial_number': 'string',
+    'location': 'string',
+    'created_at': 'string',
+    'updated_at': 'string'
+}
+
+
+class Driver:
+    def __init__(self):
+        self.stderr = None
+        self.stdout = None
+
+    def run_command(self, command):
+        args = shlex.split(command)
+        process = subprocess.Popen(args,
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+        stdout, stderr = process.communicate()
+        if stderr != '':
+            raise Exception(stderr)
+        else:
+            LOG.info("Stdout: {0}".format(stdout))
+        return stdout, stderr
+
+    def list_volumes(self, context, device_name):
+        LOG.info("Listing Volumes for {0}".format(device_name))
+        self.run_command("osdsctl volume list")
+
+    def list_pools(self, context, device_name):
+        LOG.info("Listing Pools for {0}".format(device_name))
+        self.run_command("osdsctl pool list")
+
+    def register(self):
+
+        # the real implementation
+        time.sleep(1)
+        return DEVICE_INFO

dolphin/resource_manager/storages.py

@@ -0,0 +1,98 @@
+# Copyright 2020 The SODA Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import copy
+
+from webob import exc
+from oslo_log import log
+
+from dolphin import db
+from dolphin.db.sqlalchemy import api as db
+from dolphin.driver_manager import manager as drivermanager
+
+LOG = log.getLogger(__name__)
+
+
+def build_storages(storages):
+    views = [build_storage(storage)
+             for storage in storages]
+    return dict(storage=views)
+
+
+def build_storage(storage):
+    view = copy.deepcopy(storage)
+    return view
+
+
+def get_all(req):
+    storage_all = db.storage_get_all()
+    search_opts = [
+        'name',
+        'vendor',
+        'model',
+        'status',
+    ]
+    for search_opt in search_opts:
+        if search_opt in req.GET:
+            value = req.GET[search_opt]
+            storage_all = [s for s in storage_all if s[search_opt] == value]
+        if len(storage_all) == 0:
+            break
+    return build_storages(storage_all)
+
+
+def register(self, req, body):
+    """
+            This is create function for registering the new storage device
+            :param req:
+            :param body: "It contains the all input parameters"
+            :return:
+            """
+    # Check if body is valid
+    if not self.is_valid_body(body, 'storages'):
+        msg = "Storage entity not found in request body"
+        raise exc.HTTPUnprocessableEntity(explanation=msg)
+
+    required_parameters = ('name', 'manufacturer', 'model', 'username', 'password')
+
+    storage = body['storages']
+
+    for parameter in required_parameters:
+        if parameter not in storage:
+            msg = "Required parameter %s not found" % parameter
+            raise exc.HTTPUnprocessableEntity(explanation=msg)
+        if not storage.get(parameter):
+            msg = "Required parameter %s is empty" % parameter
+            raise exc.HTTPUnprocessableEntity(explanation=msg)
+
+    driver = drivermanager.Driver()
+    device_info = driver.register()
+    if device_info.get('status') == 'available':
+        try:
+            storage['storage_id'] = device_info.get('id')
+            db.registry_context_create(storage)
+        except:
+            LOG.error('device context registration failed!!')
+            raise
+
+        try:
+            db.storage_create(device_info)
+        except:
+            LOG.error('storage DB entry creation failed')
+            raise
+    else:
+        raise Exception('device registration failed!!')
+
+    return storage

dolphin/task_manager/manager.py

@@ -1,17 +1,16 @@
-# Copyright (c) 2014 NetApp Inc.
-# All Rights Reserved.
+# Copyright 2020 The SODA Authors.
 #
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#         http://www.apache.org/licenses/LICENSE-2.0
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 """
 
 **periodical task manager**
@@ -25,7 +24,10 @@
 from dolphin import manager
 from dolphin.task_manager import rpcapi as task_rpcapi
 from dolphin import coordination
+# from dolphin.exporter import base_exporter
 from dolphin import context
+from dolphin.db.sqlalchemy import api as sqldb
+from dolphin.driver_manager import manager as dm
 
 LOG = log.getLogger(__name__)
 CONF = cfg.CONF
@@ -52,5 +54,54 @@ def say_hello(self, context, request_spec=None,
                   filter_properties=None):
         try:
             LOG.info("Consume say hello task ...")
+            # get resource data, use static data for example here
+            data = {
+                'device_id': '123456',
+                'pool_num': '4',
+            }
+            # report data to northbound platform
+            # base_exporter.dispatch_example_data(data)
+
         except Exception as ex:
             pass
+
+    def get_volumes(self, context, request_spec=None, filter_properties=None):
+        # 1. Call the list volumes
+        try:
+            driver = dm.Driver()
+            driver.list_volumes(context, request_spec)
+        except:
+            LOG.error("Volume retreival failed in driver")
+
+        # 2. Update the data to DB
+
+    def get_pools(self, context, request_spec=None, filter_properties=None):
+        try:
+            driver = dm.Driver()
+            driver.list_pools(context, request_spec)
+        except:
+            LOG.error("Pool retreival failed in driver")
+
+    def storage_device_details(self, context, req):
+        """
+         :param context:
+         :param req:
+         :return:
+        """
+        if sqldb.registry_context_get_all():
+            device_list = sqldb.registry_context_get_all()
+            for device in device_list:
+                try:
+                    self.task_rpcapi.get_pools(context, device)
+                except:
+                    LOG.error('Pools retreival failed!!')
+                    raise Exception
+
+                try:
+                    self.task_rpcapi.get_volumes(context, device)
+                except:
+                    LOG.error('Volumes retreival failed!!')
+                    raise Exception
+        else:
+            return LOG.info('There is no registered device available')
+        return

dolphin/task_manager/rpcapi.py

@@ -1,16 +1,16 @@
-# Copyright 2012, Red Hat, Inc.
+# Copyright 2020 The SODA Authors.
 #
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-#         http://www.apache.org/licenses/LICENSE-2.0
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 """
 Client side of the task manager RPC API.
@@ -49,3 +49,19 @@ def say_hello(self, context, request_spec=None,
                                  'say_hello',
                                  request_spec=request_spec_p,
                                  filter_properties=filter_properties)
+
+    def get_volumes(self, context, request_spec=None, filter_properties=None):
+        request_spec_p = jsonutils.to_primitive(request_spec)
+        call_context = self.client.prepare(version='1.0')
+        return call_context.cast(context, 'get_volumes',
+                                 request_spec=request_spec_p,
+                                 filter_properties=filter_properties
+                                 )
+
+    def get_pools(self, context, request_spec=None, filter_properties=None):
+        request_spec_p = jsonutils.to_primitive(request_spec)
+        call_context = self.client.prepare(version='1.0')
+        return call_context.cast(context, 'get_pools',
+                                 request_spec=request_spec_p,
+                                 filter_properties=filter_properties
+                                 )