cmd-cloud-prune: GC images/builds for builds

Extend the garbage collection to the images and whole builds. We
will prune all the images apart from what is specified in the
images_keep list for each stream in gc-policy.yaml. For pruning
the whole builds, we will delete all the resources in s3 for that
build and add those builds under tombstone-builds in the
respective builds.json

src/cmd-cloud-prune

@@ -22,10 +22,11 @@
 # "arches": [
 # "x86_64"
 # ],
-# "policy-cleanup": [
-# "cloud-uploads",
+# "policy-cleanup": {
+# "cloud-uploads": true,
+# "images": true,
 # "images-kept": ["qemu", "live-iso"]
-# ]
+# }
 # }
 #
 # We should also prune unreferenced build directories here. See also
@@ -40,6 +41,7 @@ import collections
 import datetime
 import os
 import boto3
+import botocore
 from dateutil.relativedelta import relativedelta
 from cosalib.gcp import remove_gcp_image
 from cosalib.aws import deregister_aws_resource
@@ -114,14 +116,24 @@ def main():
  continue
  duration = convert_duration_to_days(policy[stream][action])
  ref_date = today_date - relativedelta(days=int(duration))
+ pruned_build_ids = []
+ images_to_keep = policy.get(stream, {}).get("images-keep", [])
 
  print(f"Pruning resources of type {action} older than {policy[stream][action]} ({ref_date.date()}) on stream {stream}")
  # Enumerating in reverse to go from the oldest build to the newest one
  for build in reversed(builds):
  build_id = build["id"]
- if action in build.get("policy-cleanup", []):
- print(f"Build {build_id} has already had {action} pruning completed")
- continue
+ if action in build.get("policy-cleanup", {}):
+ if action == "images":
+ images_kept = build.get("policy-cleanup", {}).get("images-kept", [])
+ # If images in images-keep in the policy and the images_kept in build matches
+ # let's NOT run the pruning on images and mark it as completed.
+ if set(images_to_keep) == set(images_kept):
+ print(f"Build {build_id} has already had {action} pruning completed")
+ continue
+ else:
+ print(f"Build {build_id} has already had {action} pruning completed")
+ continue
  (build_date, _) = parse_fcos_version_to_timestamp_and_stream(build_id)
 
  if build_date >= ref_date:
@@ -136,14 +148,34 @@ def main():
  match action:
  case "cloud-uploads":
  prune_cloud_uploads(current_build, cloud_config, args.dry_run)
- case "build":
- raise NotImplementedError
- # print(f"Deleting key {prefix}{build.id} from bucket {bucket}")
- # Delete the build's directory in S3
- # S3().delete_object(args.bucket, f"{args.prefix}{str(current_build.id)}")
+ # Prune through images that are not mentioned in images-keep
  case "images":
- raise NotImplementedError
- build.setdefault("policy-cleanup", []).append("cloud-uploads")
+ prune_images(s3_client, current_build, images_to_keep, args.dry_run, bucket, prefix)
+ # Fully prune releases that are very old including deleting the directory in s3 for that build.
+ case "build":
+ prune_build(s3_client, bucket, prefix, build_id, args.dry_run)
+ pruned_build_ids.append(build_id)
+ # Update policy-cleanup after processing all arches for the build
+ match action:
+ case "cloud-uploads":
+ policy_cleanup = build.setdefault("policy-cleanup", {})
+ if "cloud-uploads" not in policy_cleanup:
+ policy_cleanup["cloud-uploads"] = True
+ case "images":
+ policy_cleanup = build.setdefault("policy-cleanup", {})
+ if "images" not in policy_cleanup:
+ policy_cleanup["images"] = True
+ policy_cleanup["images-kept"] = images_to_keep
+
+ if pruned_build_ids:
+ if "tombstone-builds" not in builds_json_data:
+ builds_json_data["tombstone-builds"] = []
+ # Separate the builds into remaining builds and tombstone builds
+ remaining_builds = [build for build in builds if build["id"] not in pruned_build_ids]
+ tombstone_builds = [build for build in builds if build["id"] in pruned_build_ids]
+ # Update the data structure
+ builds_json_data["builds"] = remaining_builds
+ builds_json_data["tombstone-builds"].extend(tombstone_builds)
 
  # Save the updated builds.json to local builds/builds.json
  save_builds_json(builds_json_data, BUILDFILES['list'])
@@ -320,5 +352,60 @@ def delete_gcp_image(build, cloud_config, dry_run):
  return errors
 
 
+def prune_images(s3, build, images_to_keep, dry_run, bucket, prefix):
+ images_from_meta_json = build.meta_json.get("images", [])
+ # Get the image names and paths currently in meta.json
+ current_images_data = [(name, data.get("path")) for name, data in images_from_meta_json.items()]
+ # Extract names to compare with images_to_keep
+ current_images = [name for name, _ in current_images_data]
+ errors = []
+
+ if set(images_to_keep) == set(current_images):
+ # if images_to_keep and current_images are the same, we don't need to prune anything so conitnue.
+ print(f"No images to be pruned for {build.id}")
+ return
+
+ for name, path in current_images_data:
+ if name not in images_to_keep:
+ image_prefix = os.path.join(prefix, f"{build.id}/{build.arch}/{path}")
+ if dry_run:
+ print(f"Would prune {bucket}/{image_prefix}")
+ else:
+ try:
+ s3.delete_object(Bucket=bucket, Key=image_prefix)
+ print(f"Pruned {name}")
+ except botocore.exceptions.ClientError as e:
+ if e.response['Error']['Code'] == 'NoSuchKey':
+ print(f"{bucket}/{image_prefix} already pruned.")
+ else:
+ errors.append(e)
+ if errors:
+ print(f"Found errors when pruning images for {build.id}:")
+ for e in errors:
+ print(e)
+ raise Exception("Some errors were encountered")
+
+
+def prune_build(bucket, prefix, build_id, dry_run, s3_client):
+ build_prefix = os.path.join(prefix, f"{build_id}/")
+ if dry_run:
+ print(f"Would delete all resources in {bucket}/{build_prefix}.")
+ else:
+ try:
+ objects_to_delete = s3_client.list_objects_v2(Bucket=bucket, Prefix=build_prefix)
+ if 'Contents' in objects_to_delete:
+ delete_keys = [{'Key': obj['Key']} for obj in objects_to_delete['Contents']]
+ s3_client.delete_objects(Bucket=bucket, Delete={'Objects': delete_keys})
+ print(f"Deleted all objects in {build_prefix}.")
+ else:
+ print(f"No objects found under {build_prefix}.")
+ print(f"Pruned {build_id}")
+ except botocore.exceptions.ClientError as e:
+ if e.response['Error']['Code'] == 'NoSuchKey':
+ print(f"{bucket}/{build_prefix} already pruned.")
+ else:
+ raise Exception(f"Error pruning {build_id}: {e.response['Error']['Message']}")
+
+
 if __name__ == "__main__":
  main()