cmd-cloud-prune: Prune images for builds

Extend the garbage collection to the images for the builds. We
will prune all the images apart from what is specified in the
images_keep list for each stream in gc-policy.yaml

src/cmd-cloud-prune

@@ -22,10 +22,11 @@
 # "arches": [
 # "x86_64"
 # ],
-# "policy-cleanup": [
-# "cloud-uploads",
+# "policy-cleanup": {
+# "cloud-uploads": True,
+# "images": True,
 # "images-kept": ["qemu", "live-iso"]
-# ]
+# }
 # }
 #
 # We should also prune unreferenced build directories here. See also
@@ -40,6 +41,7 @@ import collections
 import datetime
 import os
 import boto3
+import botocore
 from dateutil.relativedelta import relativedelta
 from cosalib.gcp import remove_gcp_image
 from cosalib.aws import deregister_aws_resource
@@ -119,9 +121,15 @@ def main():
  # Enumerating in reverse to go from the oldest build to the newest one
  for build in reversed(builds):
  build_id = build["id"]
- if action in build.get("policy-cleanup", []):
- print(f"Build {build_id} has already had {action} pruning completed")
- continue
+ if action in build.get("policy-cleanup", {}):
+ if action == "images":
+ # If images in images-keep in the policy and the images_kept in build doesn't match
+ # let's run the pruning again on new iamges and update the images-kept.
+ if policy.get(stream, {}).get("images-keep", []) != build["policy-cleanup"]["images-kept"]:
+ pass
+ else:
+ print(f"Build {build_id} has already had {action} pruning completed")
+ continue
  (build_date, _) = parse_fcos_version_to_timestamp_and_stream(build_id)
 
  if build_date >= ref_date:
@@ -136,14 +144,25 @@ def main():
  match action:
  case "cloud-uploads":
  prune_cloud_uploads(current_build, cloud_config, args.dry_run)
- case "build":
- raise NotImplementedError
- # print(f"Deleting key {prefix}{build.id} from bucket {bucket}")
- # Delete the build's directory in S3
- # S3().delete_object(args.bucket, f"{args.prefix}{str(current_build.id)}")
+ # Prune through images that are not mentioned in images-keep
  case "images":
+ images_to_keep = policy.get(stream, {}).get("images-keep", [])
+ prune_images(s3_client, current_build, images_to_keep, args.dry_run, bucket, prefix)
+ # Fully prune releases that are very old including deleting the directory in s3 for that build.
+ case "build":
  raise NotImplementedError
- build.setdefault("policy-cleanup", []).append("cloud-uploads")
+ # Update policy-cleanup after processing all arches for the build
+ match action:
+ case "cloud-uploads":
+ policy_cleanup = build.setdefault("policy-cleanup", {})
+ if "cloud-uploads" not in policy_cleanup:
+ policy_cleanup["cloud-uploads"] = True
+ case "images":
+ policy_cleanup = build.setdefault("policy-cleanup", {})
+ if "images" not in policy_cleanup:
+ policy_cleanup["images"] = True
+ policy_cleanup["images-kept"] = images_to_keep
+
 
  # Save the updated builds.json to local builds/builds.json
  save_builds_json(builds_json_data, BUILDFILES['list'])
@@ -320,5 +339,39 @@ def delete_gcp_image(build, cloud_config, dry_run):
  return errors
 
 
+def prune_images(s3, build, images_to_keep, dry_run, bucket, prefix):
+ images_from_meta_json = build.meta_json.get("images", [])
+ # Get the image names and paths currently in meta.json
+ current_images_data = [(name, data.get("path")) for name, data in images_from_meta_json.items()]
+ # Extract names to compare with images_to_keep
+ current_images = [name for name, _ in current_images_data]
+ errors = []
+
+ if images_to_keep == current_images:
+ # if images_to_keep and current_images are the same, we don't need to prune anything so conitnue.
+ print(f"No images to be pruned for {build.id}")
+ return
+
+ for name, path in current_images_data:
+ if name and name not in images_to_keep:
+ image_prefix = os.path.join(prefix, f"{build.id}/{build.arch}/{path}")
+ if dry_run:
+ print(f"Would prune {bucket}/{image_prefix}")
+ else:
+ try:
+ s3.delete_object(Bucket=bucket, Key=image_prefix)
+ print(f"Pruned {name}")
+ except botocore.exceptions.ClientError as e:
+ if e.response['Error']['Code'] == 'NoSuchKey':
+ print(f"{bucket}/{image_prefix} already pruned.")
+ else:
+ errors.append(e)
+ if errors:
+ print(f"Found errors when pruning images for {build.id}:")
+ for e in errors:
+ print(e)
+ raise Exception("Some errors were encountered")
+
+
 if __name__ == "__main__":
  main()