add Powerstore file system

delfin/common/constants.py

@@ -602,6 +602,7 @@ class FileSystemMetric:
     THROUGHPUT = Metrics.THROUGHPUT
     READ_THROUGHPUT = Metrics.READ_THROUGHPUT
     WRITE_THROUGHPUT = Metrics.WRITE_THROUGHPUT
+    RESPONSE_TIME = Metrics.RESPONSE_TIME
     READ_RESPONSE_TIME = Metrics.READ_RESPONSE_TIME
     WRITE_RESPONSE_TIME = Metrics.WRITE_RESPONSE_TIME
     IO_SIZE = Metrics.IO_SIZE

delfin/drivers/dell_emc/power_store/consts.py

@@ -155,6 +155,7 @@ class DiskType(object):
 PERFORMANCE_METRICS_BY_VOLUME = 'performance_metrics_by_volume'
 PERFORMANCE_METRICS_BY_NODE = 'performance_metrics_by_node'
 PERFORMANCE_METRICS_BY_FE_FC_PORT = 'performance_metrics_by_fe_fc_port'
+PERFORMANCE_METRICS_BY_FILE_SYSTEM = 'performance_metrics_by_file_system'
 PERFORMANCE_METRICS_INTERVAL = 'Twenty_Sec'
 PERF_INTERVAL = 20
 
@@ -452,3 +453,67 @@ class DiskType(object):
         "description": constants.PortMetric.WRITE_IO_SIZE.description
     }
 }
+FILE_SYSTEM_CAP = {
+    constants.FileSystemMetric.IOPS.name: {
+        "unit": constants.FileSystemMetric.IOPS.unit,
+        "description": constants.FileSystemMetric.IOPS.description
+    },
+    constants.FileSystemMetric.READ_IOPS.name: {
+        "unit": constants.FileSystemMetric.READ_IOPS.unit,
+        "description": constants.FileSystemMetric.READ_IOPS.description
+    },
+    constants.FileSystemMetric.WRITE_IOPS.name: {
+        "unit": constants.FileSystemMetric.WRITE_IOPS.unit,
+        "description": constants.FileSystemMetric.WRITE_IOPS.description
+    },
+    constants.FileSystemMetric.THROUGHPUT.name: {
+        "unit": constants.FileSystemMetric.THROUGHPUT.unit,
+        "description": constants.FileSystemMetric.THROUGHPUT.description
+    },
+    constants.FileSystemMetric.READ_THROUGHPUT.name: {
+        "unit": constants.FileSystemMetric.READ_THROUGHPUT.unit,
+        "description": constants.FileSystemMetric.READ_THROUGHPUT.description
+    },
+    constants.FileSystemMetric.WRITE_THROUGHPUT.name: {
+        "unit": constants.FileSystemMetric.WRITE_THROUGHPUT.unit,
+        "description": constants.FileSystemMetric.WRITE_THROUGHPUT.description
+    },
+    constants.FileSystemMetric.RESPONSE_TIME.name: {
+        "unit": constants.FileSystemMetric.RESPONSE_TIME.unit,
+        "description": constants.FileSystemMetric.RESPONSE_TIME.description
+    },
+    constants.FileSystemMetric.READ_RESPONSE_TIME.name: {
+        "unit": constants.FileSystemMetric.READ_RESPONSE_TIME.unit,
+        "description":
+            constants.FileSystemMetric.READ_RESPONSE_TIME.description
+    },
+    constants.FileSystemMetric.WRITE_RESPONSE_TIME.name: {
+        "unit": constants.FileSystemMetric.WRITE_RESPONSE_TIME.unit,
+        "description":
+            constants.FileSystemMetric.WRITE_RESPONSE_TIME.description
+    },
+    constants.FileSystemMetric.IO_SIZE.name: {
+        "unit": constants.FileSystemMetric.IO_SIZE.unit,
+        "description": constants.FileSystemMetric.IO_SIZE.description
+    },
+    constants.FileSystemMetric.READ_IO_SIZE.name: {
+        "unit": constants.FileSystemMetric.READ_IO_SIZE.unit,
+        "description": constants.FileSystemMetric.READ_IO_SIZE.description
+    },
+    constants.FileSystemMetric.WRITE_IO_SIZE.name: {
+        "unit": constants.FileSystemMetric.WRITE_IO_SIZE.unit,
+        "description": constants.FileSystemMetric.WRITE_IO_SIZE.description
+    }
+}
+FS_TYPE_MAP = {
+    'Primary': constants.FSType.THIN
+}
+FS_SECURITY_MODE_MAP = {
+    'Native': constants.NASSecurityMode.NATIVE,
+    'UNIX': constants.NASSecurityMode.UNIX,
+    'Windows': constants.NASSecurityMode.NTFS
+}
+POOL_MODE_MAP = {
+    'Unified': constants.StorageType.UNIFIED,
+    'Block optimized': constants.StorageType.BLOCK
+}

delfin/drivers/dell_emc/power_store/power_store.py

@@ -139,6 +139,15 @@ def collect_perf_metrics(self, context, storage_id, resource_metrics,
             LOG.info('The system(storage_id: %s) stop to collect port'
                      ' performance, The length is: %s',
                      storage_id, len(fc_port_metrics))
+        if resource_metrics.get(constants.ResourceType.FILESYSTEM):
+            file_system_metrics = self.rest_handler.get_file_system_metrics(
+                storage_id,
+                resource_metrics.get(constants.ResourceType.FILESYSTEM),
+                start_time, end_time)
+            metrics.extend(file_system_metrics)
+            LOG.info('The system(storage_id: %s) stop to collect file_system '
+                     'performance, The length is: %s',
+                     storage_id, len(file_system_metrics))
         return metrics
 
     @staticmethod
@@ -150,7 +159,8 @@ def get_capabilities(context, filters=None):
                 constants.ResourceType.STORAGE_POOL: consts.STORAGE_POOL_CAP,
                 constants.ResourceType.VOLUME: consts.VOLUME_CAP,
                 constants.ResourceType.CONTROLLER: consts.CONTROLLER_CAP,
-                constants.ResourceType.PORT: consts.PORT_CAP
+                constants.ResourceType.PORT: consts.PORT_CAP,
+                constants.ResourceType.FILESYSTEM: consts.FILE_SYSTEM_CAP
             }
         }
 
@@ -171,3 +181,15 @@ def list_volume_groups(self, context):
 
     def list_masking_views(self, context):
         return self.rest_handler.list_masking_views(self.storage_id)
+
+    def list_filesystems(self, context):
+        return self.rest_handler.list_filesystems(self.storage_id)
+
+    def list_quotas(self, context):
+        return self.rest_handler.list_quotas(self.storage_id)
+
+    def list_qtrees(self, context):
+        return self.rest_handler.list_qtrees(self.storage_id)
+
+    def list_shares(self, context):
+        return self.rest_handler.list_shares(self.storage_id)

delfin/drivers/dell_emc/power_store/rest_handler.py

@@ -37,7 +37,7 @@ class RestHandler(RestClient):
     REST_CLUSTER_URL = \
         '/api/rest/cluster?select=name,compatibility_level,global_id,state,' \
         'primary_appliance_id,id,system_time&limit=2000&offset={}'
-    REST_APPLIANCE_URL = '/api/rest/appliance?select=id,name,model' \
+    REST_APPLIANCE_URL = '/api/rest/appliance?select=id,name,model,mode' \
                          '&limit=2000&offset={}'
     REST_SOFTWARE_INSTALLED_URL = \
         '/api/rest/software_installed?select=id,release_version,' \
@@ -86,11 +86,25 @@ class RestHandler(RestClient):
         '/api/rest/ip_pool_address?select=id,name,address,appliance_id,' \
         'node_id,purposes&limit=2000&offset={}'
     REST_METRICS_ARCHIVE_URL = '/api/rest/metrics_archive'
-    REST_FILE_SYSTEM_URL = '/api/rest/file_system'
-    REST_FILE_TREE_QUOTA_URL = '/api/rest/file_tree_quota'
-    REST_SMB_SHARE_URL = '/api/rest/smb_share'
-    REST_NFS_SERVER_URL = '/api/rest/nfs_server'
-    REST_FILE_USER_QUOTA_URL = '/api/rest/file_user_quota'
+    REST_FILE_SYSTEM_URL = \
+        '/api/rest/file_system?select=id,name,access_policy,filesystem_type,' \
+        'size_total,size_used,nas_server_id&limit=2000&offset={}'
+    REST_FILE_TREE_QUOTA_URL = \
+        '/api/rest/file_tree_quota?select=id,file_system_id,path,' \
+        'description,is_user_quotas_enforced,state,hard_limit,soft_limit,' \
+        'size_used&limit=2000&offset={}'
+    REST_SMB_SHARE_URL = \
+        '/api/rest/smb_share?select=id,name,file_system_id,path' \
+        '&limit=2000&offset={}'
+    REST_NFS_EXPORT_URL = \
+        '/api/rest/nfs_export?select=id,name,description,file_system,path' \
+        '&limit=2000&offset={}'
+    REST_NAS_SERVER_URL = \
+        '/api/rest/nas_server?select=id,name&limit=2000&offset={}'
+    REST_FILE_USER_QUOTA_URL = \
+        '/api/rest/file_user_quota?select=id,uid,hard_limit,size_used,' \
+        'soft_limit,state,tree_quota_id,unix_name,windows_name,windows_sid,' \
+        'file_system_id&limit=2000&offset={}'
     AUTH_KEY = 'DELL-EMC-TOKEN'
 
     def __init__(self, **kwargs):
@@ -235,7 +249,8 @@ def get_storage_pools(self, storage_id):
                 'storage_id': storage_id,
                 'native_storage_pool_id': appliance_id,
                 'status': constants.StoragePoolStatus.NORMAL,
-                'storage_type': constants.StorageType.BLOCK,
+                'storage_type': consts.POOL_MODE_MAP.get(
+                    appliance.get('mode'), constants.StorageType.BLOCK),
                 'total_capacity': total_capacity,
                 'used_capacity': used_capacity,
                 'free_capacity': total_capacity - used_capacity
@@ -885,6 +900,29 @@ def get_fc_port_metrics(self, storage_id, resource_metrics, start_time,
             fc_port_metrics_list.extend(fc_port_metrics)
         return fc_port_metrics_list
 
+    def get_file_system_metrics(self, storage_id, resource_metrics, start_time,
+                                end_time):
+        file_system_metrics_list = []
+        file_systems = self.rest_call(self.REST_FILE_SYSTEM_URL)
+        for file_system in file_systems:
+            fs_type = file_system.get('filesystem_type')
+            if 'Snapshot' == fs_type:
+                continue
+            file_system_id = file_system.get('id')
+            file_system_name = file_system.get('name')
+            if not file_system_id or not file_system_name:
+                continue
+            data = {'entity': consts.PERFORMANCE_METRICS_BY_FILE_SYSTEM,
+                    'entity_id': file_system_id,
+                    'interval': consts.PERFORMANCE_METRICS_INTERVAL}
+            packaging_data = self.package_data(data, end_time, start_time)
+            file_system_metrics = self.set_metrics_data(
+                file_system_id, file_system_name, packaging_data,
+                resource_metrics, constants.ResourceType.FILESYSTEM,
+                storage_id)
+            file_system_metrics_list.extend(file_system_metrics)
+        return file_system_metrics_list
+
     @staticmethod
     def set_metrics_data(resource_id, resource_name, packaging_data,
                          resource_metrics, resource_type, storage_id):
@@ -940,6 +978,7 @@ def package_data(self, data, end_time, start_time):
                     continue
                 duplicate.add(about_timestamp)
                 cpu_utilization = perf.get('io_workload_cpu_utilization')
+                avg_io_size = perf.get('avg_io_size')
                 metrics_d = {
                     'iops': Decimal(str(perf.get('total_iops'))).quantize(
                         Decimal('0'), rounding="ROUND_HALF_UP"),
@@ -959,7 +998,8 @@ def package_data(self, data, end_time, start_time):
                         perf.get('avg_read_latency') / units.k, 3),
                     "writeResponseTime": round(
                         perf.get('avg_write_latency') / units.k, 3),
-                    "ioSize": round(perf.get('avg_io_size') / units.Ki, 3),
+                    "ioSize": round(perf.get('avg_size') / units.Ki, 3) if
+                    avg_io_size is None else round(avg_io_size / units.Ki, 3),
                     "readIoSize": round(
                         perf.get('avg_read_size') / units.Ki, 3),
                     "writeIoSize": round(
@@ -983,3 +1023,147 @@ def get_system_time(self):
             timestamp = int((timestamp_s + time_difference) * units.k)\
                 if system_time else None
             return timestamp
+
+    def list_filesystems(self, storage_id):
+        list_filesystems = []
+        file_system_list = self.rest_call(self.REST_FILE_SYSTEM_URL)
+        for file_system in file_system_list:
+            fs_type = file_system.get('filesystem_type')
+            if 'Snapshot' == fs_type:
+                continue
+            total_capacity = int(file_system.get('size_total'))
+            used_capacity = int(file_system.get('size_used'))
+            file_dict = {
+                'native_filesystem_id': file_system.get('id'),
+                'name': file_system.get('name'),
+                'type': consts.FS_TYPE_MAP.get(fs_type, constants.FSType.THIN),
+                'status': constants.FilesystemStatus.NORMAL,
+                'storage_id': storage_id,
+                'total_capacity': total_capacity,
+                'used_capacity': used_capacity,
+                'free_capacity': total_capacity - used_capacity,
+                'security_mode': consts.FS_SECURITY_MODE_MAP.get(
+                    file_system.get('access_policy'),
+                    constants.NASSecurityMode.MIXED)
+            }
+            list_filesystems.append(file_dict)
+        return list_filesystems
+
+    def list_qtrees(self, storage_id):
+        list_qtrees = []
+        nas_dict = {}
+        nas_server_list = self.rest_call(self.REST_NAS_SERVER_URL)
+        for nas_server in nas_server_list:
+            nas_server_id = nas_server.get('id')
+            nas_server_name = nas_server.get('name')
+            nas_dict[nas_server_id] = nas_server_name
+        file_system_list = self.rest_call(self.REST_FILE_SYSTEM_URL)
+        for file_system in file_system_list:
+            fs_type = file_system.get('filesystem_type')
+            if 'Snapshot' == fs_type:
+                continue
+            file_system_id = file_system.get('id')
+            file_system_name = file_system.get('name')
+            nas_server_id = file_system.get('nas_server_id')
+            nas_server_name = nas_dict.get(nas_server_id)
+            native_qtree_id = f'{nas_server_id}{file_system_id}'
+            name = f'NAS Servers Name:{nas_server_name}' \
+                   f'@File Systems Name:{file_system_name}'
+            qtrees_dict = {
+                'native_qtree_id': hashlib.md5(
+                    native_qtree_id.encode()).hexdigest(),
+                'name': name,
+                'storage_id': storage_id,
+                'native_filesystem_id': file_system_id,
+                'path': name,
+                'security_mode': consts.FS_SECURITY_MODE_MAP.get(
+                    file_system.get('access_policy'),
+                    constants.NASSecurityMode.MIXED)
+            }
+            list_qtrees.append(qtrees_dict)
+        return list_qtrees
+
+    def list_quotas(self, storage_id):
+        list_quotas = []
+        qtree_dict = self.get_qtree_id(storage_id)
+        tree_quota_list = self.rest_call(self.REST_FILE_TREE_QUOTA_URL)
+        for tree_quota in tree_quota_list:
+            file_system_id = tree_quota.get('file_system_id')
+            tree_quotas_dict = {
+                'native_quota_id': tree_quota.get('id'),
+                'type': constants.QuotaType.TREE,
+                'storage_id': storage_id,
+                'native_filesystem_id': file_system_id,
+                'native_qtree_id': qtree_dict.get(file_system_id),
+                'capacity_hard_limit': tree_quota.get('hard_limit'),
+                'capacity_soft_limit': tree_quota.get('soft_limit'),
+                'used_capacity': tree_quota.get('size_used')
+            }
+            list_quotas.append(tree_quotas_dict)
+        user_quota_list = self.rest_call(self.REST_FILE_USER_QUOTA_URL)
+        for user_quota in user_quota_list:
+            user_group_name = user_quota.get('unix_name')
+            windows_name = user_quota.get('windows_name')
+            windows_sid = user_quota.get('windows_sid')
+            uid = user_quota.get('uid')
+            if windows_name:
+                user_group_name = windows_name
+            if windows_sid:
+                user_group_name = windows_sid
+            if uid:
+                user_group_name = uid
+            file_system_id = user_quota.get('file_system_id')
+            user_quotas_dict = {
+                'native_quota_id': user_quota.get('id'),
+                'type': constants.QuotaType.USER,
+                'storage_id': storage_id,
+                'native_filesystem_id': user_quota.get('file_system_id'),
+                'native_qtree_id': qtree_dict.get(file_system_id),
+                'capacity_hard_limit': user_quota.get('hard_limit'),
+                'capacity_soft_limit': user_quota.get('soft_limit'),
+                'used_capacity': user_quota.get('size_used'),
+                'user_group_name': user_group_name
+            }
+            list_quotas.append(user_quotas_dict)
+        return list_quotas
+
+    def get_qtree_id(self, storage_id):
+        qtree_dict = {}
+        list_qtrees = self.list_qtrees(storage_id)
+        for qtrees in list_qtrees:
+            native_qtree_id = qtrees.get('native_qtree_id')
+            native_filesystem_id = qtrees.get('native_filesystem_id')
+            qtree_dict[native_filesystem_id] = native_qtree_id
+        return qtree_dict
+
+    def list_shares(self, storage_id):
+        list_shares = []
+        qtree_dict = self.get_qtree_id(storage_id)
+        nfs_list = self.rest_call(self.REST_NFS_EXPORT_URL)
+        for nfs in nfs_list:
+            file_system_id = nfs.get('file_system', {}).get('id')
+            nfs_dict = {
+                'native_share_id': nfs.get('id'),
+                'name': nfs.get('name'),
+                'storage_id': storage_id,
+                'native_filesystem_id': file_system_id,
+                'native_qtree_id': qtree_dict.get(file_system_id),
+                'protocol': constants.ShareProtocol.NFS,
+                'path': nfs.get('path')
+            }
+            list_shares.append(nfs_dict)
+        shares_list = self.rest_call(self.REST_SMB_SHARE_URL)
+        for shares in shares_list:
+            shares_path = shares.get('path')
+            file_system_id = shares.get('file_system_id')
+            shares_dict = {
+                'native_share_id': shares.get('id'),
+                'name': shares.get('name'),
+                'storage_id': storage_id,
+                'native_filesystem_id': file_system_id,
+                'native_qtree_id': qtree_dict.get(file_system_id),
+                'protocol': constants.ShareProtocol.CIFS,
+                'path': shares_path
+            }
+            list_shares.append(shares_dict)
+        return list_shares