Merge pull request #19 from shubhamsg199/cleanup_multiple_regions

Add support for deleting resources from multiple regions in AWS
RedHatQE · Jul 21, 2022 · b48e118 · b48e118
2 parents 26d7ddb + 7b8ee13
commit b48e118
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 62 deletions.
diff --git a/cloudwash/client.py b/cloudwash/client.py
@@ -7,9 +7,8 @@
 
 
 @contextmanager
-def compute_client(compute_resource):
+def compute_client(compute_resource, **kwargs):
     """The context manager for compute resource client to initiate and disconnect
-
     :param str compute_resource: The compute resource name
     """
     if compute_resource == "azure":
@@ -33,7 +32,7 @@ def compute_client(compute_resource):
         client = wrapanapi.EC2System(
             username=settings.providers.ec2.username,
             password=settings.providers.ec2.password,
-            region=settings.providers.ec2.region,
+            region=kwargs['ec2_region'],
         )
     else:
         raise ValueError(

diff --git a/cloudwash/providers/ec2.py b/cloudwash/providers/ec2.py
@@ -10,67 +10,77 @@
 def cleanup(**kwargs):
 
     is_dry_run = kwargs["dry_run"]
-
-    with compute_client("ec2") as ec2_client:
-        # Dry Data Collection Defs
-        def dry_vms():
-            all_vms = ec2_client.list_vms()
-            for vm in all_vms:
-                if vm.name in settings.providers.ec2.except_vm_list:
-                    dry_data["VMS"]["skip"].append(vm.name)
-                    continue
-                elif total_running_time(vm).minutes >= settings.sla_minutes:
-                    if vm.name in settings.providers.ec2.except_vm_stop_list:
-                        dry_data["VMS"]["stop"].append(vm.name)
+    data = ['VMS', 'DISCS', 'PIPS', 'RESOURCES']
+    regions = settings.providers.ec2.regions
+    with compute_client("ec2", ec2_region="us-west-2") as client:
+        if "all" in regions:
+            regions = client.list_regions()
+    for region in regions:
+        dry_data['VMS']['stop'] = []
+        dry_data['VMS']['skip'] = []
+        for items in data:
+            dry_data[items]['delete'] = []
+        with compute_client("ec2", ec2_region=region) as ec2_client:
+            # Dry Data Collection Defs
+            def dry_vms():
+                all_vms = ec2_client.list_vms()
+                for vm in all_vms:
+                    if vm.name in settings.providers.ec2.except_vm_list:
+                        dry_data["VMS"]["skip"].append(vm.name)
                         continue
-                    elif vm.name.startswith(settings.delete_vm):
-                        dry_data["VMS"]["delete"].append(vm.name)
-            return dry_data["VMS"]
+                    elif total_running_time(vm).minutes >= settings.sla_minutes:
+                        if vm.name in settings.providers.ec2.except_vm_stop_list:
+                            dry_data["VMS"]["stop"].append(vm.name)
+                            continue
+                        elif vm.name.startswith(settings.delete_vm):
+                            dry_data["VMS"]["delete"].append(vm.name)
+                return dry_data["VMS"]
 
-        def dry_nics():
-            rnics = ec2_client.get_all_unused_network_interfaces()
-            [dry_data["NICS"]["delete"].append(dnic["NetworkInterfaceId"]) for dnic in rnics]
-            return dry_data["NICS"]["delete"]
+            def dry_nics():
+                rnics = ec2_client.get_all_unused_network_interfaces()
+                [dry_data["NICS"]["delete"].append(dnic["NetworkInterfaceId"]) for dnic in rnics]
+                return dry_data["NICS"]["delete"]
 
-        def dry_discs():
-            rdiscs = ec2_client.get_all_unattached_volumes()
-            [dry_data["DISCS"]["delete"].append(ddisc["VolumeId"]) for ddisc in rdiscs]
-            return dry_data["DISCS"]["delete"]
+            def dry_discs():
+                rdiscs = ec2_client.get_all_unattached_volumes()
+                [dry_data["DISCS"]["delete"].append(ddisc["VolumeId"]) for ddisc in rdiscs]
+                return dry_data["DISCS"]["delete"]
 
-        def dry_pips():
-            rpips = ec2_client.get_all_disassociated_addresses()
-            [dry_data["PIPS"]["delete"].append(dpip["AllocationId"]) for dpip in rpips]
-            return dry_data["PIPS"]["delete"]
+            def dry_pips():
+                rpips = ec2_client.get_all_disassociated_addresses()
+                [dry_data["PIPS"]["delete"].append(dpip["AllocationId"]) for dpip in rpips]
+                return dry_data["PIPS"]["delete"]
 
-        # Remove / Stop VMs
-        def remove_vms(avms):
-            # Remove VMs
-            [ec2_client.get_vm(vm_name).delete() for vm_name in avms["delete"]]
-            # Stop VMs
-            [ec2_client.get_vm(vm_name).stop() for vm_name in avms["stop"]]
+            # Remove / Stop VMs
+            def remove_vms(avms):
+                # Remove VMs
+                [ec2_client.get_vm(vm_name).delete() for vm_name in avms["delete"]]
+                # Stop VMs
+                [ec2_client.get_vm(vm_name).stop() for vm_name in avms["stop"]]
 
-        # Actual Cleaning and dry execution
-        if kwargs["vms"] or kwargs["_all"]:
-            avms = dry_vms()
-            if not is_dry_run:
-                remove_vms(avms=avms)
-                logger.info(f"Stopped VMs: \n{avms['stop']}")
-                logger.info(f"Removed VMs: \n{avms['delete']}")
-                logger.info(f"Skipped VMs: \n{avms['skip']}")
-        if kwargs["nics"] or kwargs["_all"]:
-            rnics = dry_nics()
-            if not is_dry_run:
-                ec2_client.remove_all_unused_nics()
-                logger.info(f"Removed NICs: \n{rnics}")
-        if kwargs["discs"] or kwargs["_all"]:
-            rdiscs = dry_discs()
-            if not is_dry_run:
-                ec2_client.remove_all_unused_volumes()
-                logger.info(f"Removed Discs: \n{rdiscs}")
-        if kwargs["pips"] or kwargs["_all"]:
-            rpips = dry_pips()
-            if not is_dry_run:
-                ec2_client.remove_all_unused_ips()
-                logger.info(f"Removed PIPs: \n{rpips}")
-        if is_dry_run:
-            echo_dry(dry_data)
+            # Actual Cleaning and dry execution
+            logger.info(f"\nResources from the region: {region}")
+            if kwargs["vms"] or kwargs["_all"]:
+                avms = dry_vms()
+                if not is_dry_run:
+                    remove_vms(avms=avms)
+                    logger.info(f"Stopped VMs: \n{avms['stop']}")
+                    logger.info(f"Removed VMs: \n{avms['delete']}")
+                    logger.info(f"Skipped VMs: \n{avms['skip']}")
+            if kwargs["nics"] or kwargs["_all"]:
+                rnics = dry_nics()
+                if not is_dry_run:
+                    ec2_client.remove_all_unused_nics()
+                    logger.info(f"Removed NICs: \n{rnics}")
+            if kwargs["discs"] or kwargs["_all"]:
+                rdiscs = dry_discs()
+                if not is_dry_run:
+                    ec2_client.remove_all_unused_volumes()
+                    logger.info(f"Removed Discs: \n{rdiscs}")
+            if kwargs["pips"] or kwargs["_all"]:
+                rpips = dry_pips()
+                if not is_dry_run:
+                    ec2_client.remove_all_unused_ips()
+                    logger.info(f"Removed PIPs: \n{rpips}")
+            if is_dry_run:
+                echo_dry(dry_data)
diff --git a/settings.yaml.template b/settings.yaml.template
@@ -28,7 +28,8 @@ PROVIDERS:
   EC2:
     USERNAME:
     PASSWORD:
-    REGION:
+    # Multiple regions can be added like ["ap-south-1", "us-west-2", "us-west-1"] or ["all"] for all regions
+    REGIONS: []
     # VMs that would be skipped from cleanup
     EXCEPT_VM_LIST: []
     # VMs that would be stopped from running states