gce: Cleanup buckets

ocw/lib/gce.py

@@ -6,7 +6,7 @@
 import googleapiclient.discovery
 from googleapiclient.errors import HttpError
 from google.oauth2 import service_account
-from webui.PCWConfig import ConfigFile
+from webui.PCWConfig import ConfigFile, PCWConfig
 from .provider import Provider
 
 
@@ -21,9 +21,11 @@ def __new__(cls, namespace):
     def __init__(self, namespace):
         super().__init__(namespace)
 
+        self.__bucket = PCWConfig.get_feature_property('cleanup', 'gce-bucket', namespace)
         self.__skip_networks = frozenset(ConfigFile().getList('cleanup/gce-skip-networks', ["default"]))
 
         self.__compute_client = None
+        self.__storage_client = None
         self.private_key_data = self.get_data()
         self.project = self.private_key_data["project_id"]
 
@@ -41,6 +43,7 @@ def _paginated(self, api_call, **kwargs) -> list:
 
     def _delete_resource(self, api_call, resource_name, *_, **kwargs) -> None:
         resource_type = {
+            self.storage_client().objects: "blob",
             self.compute_client().disks: "disk",
             self.compute_client().firewalls: "firewall",
             self.compute_client().forwardingRules: "forwardingRule",
@@ -87,6 +90,14 @@ def compute_client(self):
             )
         return self.__compute_client
 
+    def storage_client(self):
+        if self.__storage_client is None:
+            credentials = service_account.Credentials.from_service_account_info(self.private_key_data)
+            self.__storage_client = googleapiclient.discovery.build(
+                "storage", "v1", credentials=credentials, cache_discovery=False
+            )
+        return self.__storage_client
+
     def list_instances(self, zone) -> list:
         """ List all instances by zone."""
         self.log_dbg(f"Call list_instances for {zone}")
@@ -134,6 +145,7 @@ def get_error_reason(error: "googleapiclient.errors.HttpError") -> str:
 
     def cleanup_all(self) -> None:
         self.log_info("Call cleanup_all")
+        self.cleanup_blobs()
         self.cleanup_disks()
         self.cleanup_images()
         self.cleanup_firewalls()
@@ -142,6 +154,16 @@ def cleanup_all(self) -> None:
         self.cleanup_subnetworks()
         self.cleanup_networks()
 
+    def cleanup_blobs(self) -> None:
+        self.log_dbg("Blobs cleanup")
+        blobs = self._paginated(self.storage_client().objects, bucket=self.__bucket)
+        self.log_dbg(f"{len(blobs)} blobs found")
+        for blob in blobs:
+            if self.is_outdated(parse(blob["timeCreated"]).astimezone(timezone.utc)):
+                self._delete_resource(
+                    self.storage_client().objects, blob["name"], bucket=self.__bucket, object=blob["name"]
+                )
+
     def cleanup_disks(self) -> None:
         self.log_dbg("Disks cleanup")
         for region in self.list_regions():

templates/pcw.ini

@@ -56,6 +56,8 @@ azure-storage-resourcegroup = openqa-upload
 azure-storage-account-name = openqa
 # When set to true EC2 VPC cleanup will be enabled
 vpc_cleanup = true
+# GCE bucket to be cleaned up
+gce_bucket = bucket
 
 [updaterun]
 # if openqa_ttl tag is not defined this TTL will be set to the instance

webui/PCWConfig.py

@@ -55,11 +55,12 @@ class PCWConfig():
     @staticmethod
     def get_feature_property(feature: str, feature_property: str, namespace: str | None = None) -> str | int:
         default_values: dict[str, dict[str, int | type[int] | str | type[str] | type[str] | None]] = {
-            'cleanup/max-age-hours': {'default': 24 * 7, 'return_type': int},
             'cleanup/azure-gallery-name': {'default': 'test_image_gallery', 'return_type': str},
             'cleanup/azure-storage-resourcegroup': {'default': 'openqa-upload', 'return_type': str},
             'cleanup/azure-storage-account-name': {'default': 'openqa', 'return_type': str},
             'cleanup/ec2-max-age-days': {'default': -1, 'return_type': int},
+            'cleanup/gce-bucket': {'default': 'test', 'return_type': str},
+            'cleanup/max-age-hours': {'default': 24 * 7, 'return_type': int},
             'cleanup/openstack-image-max-age-days': {'default': 3, 'return_type': int},
             'cleanup/openstack-vm-max-age-days': {'default': 1, 'return_type': int},
             'cleanup/openstack-key-max-days': {'default': 1, 'return_type': int},