Merge pull request #541 from openvstorage/mds_regression_changes

Fixups for mds_regression

ci/scenarios/edge/reroute/main.py

@@ -245,7 +245,7 @@ def test_reroute_fio(cls, fio_bin_path, cluster_info, disk_amount=1, timeout=CIC
                 for thread_category, thread_collection in threads['evented'].iteritems():
                     ThreadHelper.stop_evented_threads(thread_collection['pairs'], thread_collection['r_semaphore'])
             for vdisk in vdisk_info.values():
-                VDiskRemover.remove_vdisk(vdisk.guid)
+                VDiskRemover.remove_vdisk(vdisk.guid, api)
         assert len(failed_configurations) == 0, 'Certain configuration failed: {0}'.format(failed_configurations)
 
     @staticmethod

ci/scenarios/hypervisor/automated_ha/main.py

@@ -163,9 +163,7 @@ def start_test(cls, vm_amount=1, hypervisor_info=CIConstants.HYPERVISOR_INFO):
             cls.run_test(cluster_info=cluster_info, vm_info=vm_info)
         finally:
             for vm_name, vm_object in vm_info.iteritems():
-                for vdisk in vm_object['vdisks']:
-                    VDiskRemover.remove_vdisk(vdisk.guid)
-            for vm_name in vm_info.keys():
+                VDiskRemover.remove_vdisks_with_structure(vm_object['vdisks'], api)
                 computenode_hypervisor.sdk.destroy(vm_name)
                 computenode_hypervisor.sdk.undefine(vm_name)
         # cls.test_ha_fio(fio_bin_path, cluster_info, is_ee, api)
@@ -384,7 +382,7 @@ def test_ha_fio(cls, fio_bin_path, cluster_info, is_ee,  api, disk_amount=1, tim
             for thread_category, thread_collection in threads['evented'].iteritems():
                 ThreadHelper.stop_evented_threads(thread_collection['pairs'], thread_collection['r_semaphore'])
             for vdisk in vdisk_info.values():
-                VDiskRemover.remove_vdisk(vdisk.guid)
+                VDiskRemover.remove_vdisk(vdisk.guid, api)
         assert len(failed_configurations) == 0, 'Certain configuration failed: {0}'.format(' '.join(failed_configurations))
 
     @staticmethod

ci/scenarios/hypervisor/live_migrate_vm_test-exclude/main.py

@@ -108,8 +108,7 @@ def start_test(cls, vm_amount=1, hypervisor_info=CIConstants.HYPERVISOR_INFO):
             cls.live_migrate(vm_info, cluster_info, volume_amount, hypervisor_info)
         finally:
             for vm_name, vm_object in vm_info.iteritems():
-                for vdisk in vm_object['vdisks']:
-                    VDiskRemover.remove_vdisk(vdisk.guid)
+                VDiskRemover.remove_vdisks_with_structure(vm_object['vdisks'], api)
             for vm_name in vm_info.keys():
                 source_hypervisor.sdk.destroy(vm_name)
                 source_hypervisor.sdk.undefine(vm_name)

ci/scenarios/vDisk/advanced_dtl_vdisk_test/main.py

@@ -113,9 +113,7 @@ def start_test(cls, vm_amount=1, hypervisor_info=CIConstants.HYPERVISOR_INFO):
             cls.run_test(vm_info=vm_info, cluster_info=cluster_info)
         finally:
             for vm_name, vm_object in vm_info.iteritems():
-                for vdisk in vm_object['vdisks']:
-                    VDiskRemover.remove_vdisk(vdisk.guid)
-            for vm_name in vm_info.keys():
+                VDiskRemover.remove_vdisks_with_structure(vm_object['vdisks'], api)
                 computenode_hypervisor.sdk.destroy(vm_name)
                 computenode_hypervisor.sdk.undefine(vm_name)
 

ci/scenarios/vDisk/basic_dtl_vdisk_test/main.py

@@ -13,11 +13,9 @@
 #
 # Open vStorage is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY of any kind.
-import json
 import time
 import random
-from ci.main import CONFIG_LOC
-from ci.api_lib.helpers.api import OVSClient, TimeOutError
+from ci.api_lib.helpers.api import TimeOutError
 from ci.api_lib.helpers.domain import DomainHelper
 from ci.api_lib.helpers.storagedriver import StoragedriverHelper
 from ci.api_lib.helpers.vdisk import VDiskHelper
@@ -45,7 +43,6 @@ def __init__(self):
     def main(blocked):
         """
         Run all required methods for the test
-
         :param blocked: was the test blocked by other test?
         :type blocked: bool
         :return: results of test
@@ -54,36 +51,22 @@ def main(blocked):
         _ = blocked
         return DTLChecks._execute_test()
 
-    @staticmethod
-    def _execute_test():
+    @classmethod
+    def _execute_test(cls):
         """
         Validate if DTL is configured as desired
-
         REQUIREMENTS:
         * 1 vPool should be available with 1 storagedriver
         * 1 vPool should be available with 2 or more storagedrivers in 2 separate domains
-
         OPTIONAL:
         * 1 vPool with 1 storagedriver with disabled DTL
-
         :return:
         """
-
         DTLChecks.LOGGER.info("Starting to validate the basic DTL")
-
-        with open(CONFIG_LOC, "r") as JSON_CONFIG:
-            config = json.load(JSON_CONFIG)
-
-        api = OVSClient(
-            config['ci']['grid_ip'],
-            config['ci']['user']['api']['username'],
-            config['ci']['user']['api']['password']
-        )
-
+        api = cls.get_api_instance()
         ##########################
         # get deployment details #
         ##########################
-
         vpools = VPoolHelper.get_vpools()
         assert len(vpools) >= 1, "Not enough vPools to test"
 
@@ -255,7 +238,7 @@ def _execute_test():
                         DTLChecks.LOGGER.info("Changing config to a same domain with only 1 storagedriver was successful!")
 
                     DTLChecks.LOGGER.info("Removing vDisk {0}".format(vdisk.name))
-                    VDiskRemover.remove_vdisk(vdisk_guid=vdisk.guid)
+                    VDiskRemover.remove_vdisk(vdisk_guid=vdisk.guid, api=api)
                     DTLChecks.LOGGER.info("Finished removing vDisk {0}".format(vdisk.name))
 
             end = time.time()

ci/scenarios/vDisk/data_corruption_reg_test/main.py

@@ -112,9 +112,7 @@ def start_test(cls, vm_amount=1, hypervisor_info=CIConstants.HYPERVISOR_INFO):
             cls.run_test(storagedriver=storagedriver, vm_info=vm_info)
         finally:
             for vm_name, vm_object in vm_info.iteritems():
-                for vdisk in vm_object['vdisks']:
-                    VDiskRemover.remove_vdisk(vdisk.guid)
-            for vm_name in vm_info.keys():
+                VDiskRemover.remove_vdisks_with_structure(vm_object['vdisks'], api)
                 computenode_hypervisor.sdk.destroy(vm_name)
                 computenode_hypervisor.sdk.undefine(vm_name)
 

ci/scenarios/vDisk/deployment_vdisk_test/main.py

@@ -105,7 +105,7 @@ def validate_vdisk_deployment(cls):
             VDiskDeploymentChecks.LOGGER.info("Finished creating vdisk `{0}`".format(api_disk_name))
             VDiskDeploymentChecks._check_vdisk(vdisk_name=api_disk_name, vpool_name=vpool.name)
             VDiskDeploymentChecks.LOGGER.info("Starting to delete vdisk `{0}`".format(api_disk_name))
-            VDiskRemover.remove_vdisk_by_name(api_disk_name+'.raw', vpool.name)
+            VDiskRemover.remove_vdisk_by_name(api_disk_name, vpool.name, api)
             VDiskDeploymentChecks.LOGGER.info("Finished deleting vdisk `{0}`".format(api_disk_name))
 
         # ========
@@ -123,7 +123,7 @@ def validate_vdisk_deployment(cls):
             VDiskDeploymentChecks.LOGGER.info("Finished creating vdisk `{0}`".format(qemu_disk_name))
             VDiskDeploymentChecks._check_vdisk(vdisk_name=qemu_disk_name, vpool_name=vpool.name)
             VDiskDeploymentChecks.LOGGER.info("Starting to delete vdisk `{0}`".format(qemu_disk_name))
-            VDiskRemover.remove_vdisk_by_name(qemu_disk_name+'.raw', vpool.name)
+            VDiskRemover.remove_vdisk_by_name(qemu_disk_name, vpool.name, api)
             VDiskDeploymentChecks.LOGGER.info("Finished deleting vdisk `{0}`".format(qemu_disk_name))
 
         # ============
@@ -137,7 +137,7 @@ def validate_vdisk_deployment(cls):
             VDiskDeploymentChecks.LOGGER.info("Finished creating vdisk `{0}`".format(truncate_disk_name))
             VDiskDeploymentChecks._check_vdisk(vdisk_name=truncate_disk_name, vpool_name=vpool.name)
             VDiskDeploymentChecks.LOGGER.info("Starting to delete vdisk `{0}`".format(truncate_disk_name))
-            VDiskRemover.remove_vdisk_by_name(truncate_disk_name+'.raw', vpool.name)
+            VDiskRemover.remove_vdisk_by_name(truncate_disk_name, vpool.name, api)
             VDiskDeploymentChecks.LOGGER.info("Finished deleting vdisk `{0}`".format(truncate_disk_name))
         VDiskDeploymentChecks.LOGGER.info("Finished to validate the vdisk deployment")
 

ci/scenarios/vDisk/offline_migrate_test/main.py

@@ -118,7 +118,7 @@ def _execute_test(cls, amount_vdisks=AMOUNT_VDISKS):
                         raise
         finally:
             for vdisk in created_vdisks:
-                VDiskRemover.remove_vdisk(vdisk.guid)
+                VDiskRemover.remove_vdisk(vdisk.guid, api)
         MigrateTester.LOGGER.info("Finished offline migrate test.")
 
     @staticmethod

ci/scenarios/vDisk/rapid_creation_same_device/main.py

@@ -46,14 +46,15 @@ def main(blocked):
         _ = blocked
         return VDiskControllerTester._execute_test()
 
-    @staticmethod
-    def _execute_test():
+    @classmethod
+    def _execute_test(cls):
         """
         Mimics the healthcheck creating and deleting disks with the same name/devicename back to back
         :return: None
         """
         local_sr = SystemHelper.get_local_storagerouter()
         VDiskControllerTester.LOGGER.info("Starting creation/deletion test.")
+        api = cls.get_api_instance()
         # Elect vpool
         assert len(local_sr.storagedrivers) > 0, 'Node {0} has no storagedriver. Cannot test {1}'.format(local_sr.ip, VDiskControllerTester.TEST_NAME)
         random_storagedriver = local_sr.storagedrivers[random.randint(0, len(local_sr.storagedrivers) - 1)]
@@ -89,7 +90,7 @@ def _execute_test():
                 VDiskControllerTester.LOGGER.error('Unexpected exception occurred during the the loop. Got {0}.'.format(str(ex)))
             finally:
                 try:
-                    VDiskControllerTester._cleanup_vdisk(disk_name, vpool.name, not test_passed)
+                    VDiskControllerTester._cleanup_vdisk(disk_name, vpool.name, api, not test_passed)
                 except Exception as ex:
                     VDiskControllerTester.LOGGER.error("Auto cleanup failed with {0}.".format(str(ex)))
                     exceptions.append('Auto cleanup failed. Got {0}'.format(str(ex)))
@@ -99,7 +100,7 @@ def _execute_test():
         VDiskControllerTester.LOGGER.info("Finished create/delete test.")
 
     @staticmethod
-    def _cleanup_vdisk(vdisk_name, vpool_name, fail=True):
+    def _cleanup_vdisk(vdisk_name, vpool_name, api, fail=True):
         """
         Attempt to cleanup vdisk
         :param vdisk_name: name of the vdisk
@@ -109,7 +110,7 @@ def _cleanup_vdisk(vdisk_name, vpool_name, fail=True):
         """
         # Cleanup vdisk using the controller
         try:
-            VDiskRemover.remove_vdisk_by_name('{0}.raw'.format(vdisk_name), vpool_name)
+            VDiskRemover.remove_vdisk_by_name(vdisk_name, vpool_name, api)
         except Exception as ex:
             VDiskControllerTester.LOGGER.error(str(ex))
             if fail is True:

ci/scenarios/vDisk/regress_template_memleak_test/main.py

@@ -136,14 +136,14 @@ def validate_vdisk_clone(amount_vdisks=AMOUNT_VDISKS, amount_to_write=AMOUNT_TO_
                         "--output={0}.json".format(vdisk_name)])
             # delete vdisk
             time.sleep(VDiskTemplateChecks.TEMPLATE_SLEEP_BEFORE_DELETE)
-            VDiskRemover.remove_vdisk_by_name(vdisk_name=clone_vdisk_name + '.raw', vpool_name=vpool.name)
+            VDiskRemover.remove_vdisk_by_name(vdisk_name=clone_vdisk_name, vpool_name=vpool.name, api=api)
 
         ###################
         # remove template #
         ###################
 
         time.sleep(VDiskTemplateChecks.TEMPLATE_SLEEP_BEFORE_DELETE)
-        VDiskRemover.remove_vtemplate_by_name(vdisk_name=vdisk_name + '.raw', vpool_name=vpool.name, api=api)
+        VDiskRemover.remove_vtemplate_by_name(vdisk_name=vdisk_name, vpool_name=vpool.name, api=api)
 
         ######################
         # log current memory #

ci/scenarios/vDisk/rollback_vdisk_test/main.py

@@ -101,7 +101,7 @@ def validate_rollback():
             end = time.time()
             # clean base disks from clones
             if cloned:
-                RollbackChecks._delete_remaining_vdisks(base_vdisks=deployed_vdisks[1])
+                RollbackChecks._delete_remaining_vdisks(base_vdisks=deployed_vdisks[1], api=api)
                 RollbackChecks.LOGGER.info("Finished deleting base vdisks")
             else:
                 RollbackChecks.LOGGER.info("Skipped deleting base vdisks")
@@ -112,25 +112,24 @@ def validate_rollback():
         RollbackChecks.LOGGER.info("Finished to validate the rollback")
 
     @staticmethod
-    def _delete_remaining_vdisks(base_vdisks):
+    def _delete_remaining_vdisks(base_vdisks, api):
         """
         Delete remaining base vdisks (when performing cloned=True)
-
         :param base_vdisks: vdisk_guids of a base_vdisks ['a15908c0-f7f0-402e-ad20-2be97e401cd3', ...]
         :type: list
+        :param api: api instance
         :return: None
         """
 
         for vdisk_guid in base_vdisks:
             RollbackChecks.LOGGER.info("Starting to remove base vDisk `{0}`".format(vdisk_guid))
-            VDiskRemover.remove_vdisk(vdisk_guid)
+            VDiskRemover.remove_vdisk(vdisk_guid, api)
             RollbackChecks.LOGGER.info("Finished to remove base vDisk `{0}`".format(vdisk_guid))
 
     @staticmethod
     def _deploy_vdisks(vpool, storagedriver, api, size=SIZE_VDISK, amount_vdisks=AMOUNT_VDISKS, cloned=False):
         """
         Deploy X amount of vdisks, write some data to it & snapshot
-
         :param vpool: a valid vpool object
         :type vpool: ovs.model.hybrids.vpool
         :param storagedriver: a valid storagedriver object
@@ -290,7 +289,7 @@ def _rollback_vdisks(stored_vdisks, vpool, api, amount_checks=MAX_ROLLBACK_CHECK
 
             # commencing deleting volumes
             RollbackChecks.LOGGER.info("Starting to remove VDisk `{0}`".format(vdisk.name))
-            VDiskRemover.remove_vdisk(stored_vdisk['vdisk_guid'])
+            VDiskRemover.remove_vdisk(stored_vdisk['vdisk_guid'], api)
             RollbackChecks.LOGGER.info("Finished removing VDisk `{0}`".format(vdisk.name))
 
 

ci/scenarios/vDisk/validate_clone_disk_test/main.py

@@ -103,7 +103,7 @@ def validate_vdisk_clone(cls):
                                         snapshot_id=snapshot_id)['vdisk_guid'])
             vdisks.append(cloned_vdisk)
         finally:
-            VDiskRemover.remove_vdisks_with_structure(vdisks)
+            VDiskRemover.remove_vdisks_with_structure(vdisks, api)
         cls.LOGGER.info("Finished validating clone vdisks")
 
 

ci/scenarios/vDisk/validate_template_disk_test/main.py

@@ -104,7 +104,7 @@ def validate_vdisk_clone(cls):
         finally:
             while len(vdisks) > 0:
                 vdisk = vdisks.pop()
-                VDiskRemover.remove_vdisk(vdisk.guid)
+                VDiskRemover.remove_vdisk(vdisk.guid, api)
         try:
             # template vdisk from clone (should fail) #
             parent_vdisk = VDiskHelper.get_vdisk_by_guid(
@@ -134,9 +134,9 @@ def validate_vdisk_clone(cls):
                 if vdisk.parent_vdisk_guid is None:
                     parent_vdisks.append(vdisk)
                     continue
-                VDiskRemover.remove_vdisk(vdisk.guid)
+                VDiskRemover.remove_vdisk(vdisk.guid, api)
             for parent_vdisk in parent_vdisks:
-                VDiskRemover.remove_vdisk(parent_vdisk.guid)
+                VDiskRemover.remove_vdisk(parent_vdisk.guid, api)
         cls.LOGGER.info("Finished to validate template vdisks")
 
 

ci/scenarios/vPool/add_remove_alba_vpool_test/main.py

@@ -193,7 +193,7 @@ def validate_add_extend_remove_vpool(timeout=ADD_EXTEND_REMOVE_VPOOL_TIMEOUT):
                                     timeout=AddRemoveVPool.VDISK_CREATE_TIMEOUT)
             AddRemoveVPool.LOGGER.info("Finished creating vdisk `{0}`".format(vdisk_name))
             AddRemoveVPool.LOGGER.info("Starting to delete vdisk `{0}`".format(vdisk_name))
-            VDiskRemover.remove_vdisk_by_name(vdisk_name + '.raw', AddRemoveVPool.VPOOL_NAME)
+            VDiskRemover.remove_vdisk_by_name(vdisk_name, AddRemoveVPool.VPOOL_NAME, api)
             AddRemoveVPool.LOGGER.info("Finished deleting vdisk `{0}`".format(vdisk_name))
 
             # Delete vpool