diff --git a/src/cmd-cloud-prune b/src/cmd-cloud-prune index fbded1053b..aa4bed53d7 100755 --- a/src/cmd-cloud-prune +++ b/src/cmd-cloud-prune @@ -22,10 +22,11 @@ # "arches": [ # "x86_64" # ], -# "policy-cleanup": [ -# "cloud-uploads", +# "policy-cleanup": { +# "cloud-uploads": True, +# "images": True, # "images-kept": ["qemu", "live-iso"] -# ] +# } # } # # We should also prune unreferenced build directories here. See also @@ -40,6 +41,7 @@ import collections import datetime import os import boto3 +import botocore from dateutil.relativedelta import relativedelta from cosalib.gcp import remove_gcp_image from cosalib.aws import deregister_aws_resource @@ -119,9 +121,15 @@ def main(): # Enumerating in reverse to go from the oldest build to the newest one for build in reversed(builds): build_id = build["id"] - if action in build.get("policy-cleanup", []): - print(f"Build {build_id} has already had {action} pruning completed") - continue + if action in build.get("policy-cleanup", {}): + if action == "images": + # If images in images-keep in the policy and the images_kept in build doesn't match + # let's run the pruning again on new iamges and update the images-kept. + if policy.get(stream, {}).get("images-keep", []) != build["policy-cleanup"]["images-kept"]: + pass + else: + print(f"Build {build_id} has already had {action} pruning completed") + continue (build_date, _) = parse_fcos_version_to_timestamp_and_stream(build_id) if build_date >= ref_date: @@ -136,14 +144,25 @@ def main(): match action: case "cloud-uploads": prune_cloud_uploads(current_build, cloud_config, args.dry_run) - case "build": - raise NotImplementedError - # print(f"Deleting key {prefix}{build.id} from bucket {bucket}") - # Delete the build's directory in S3 - # S3().delete_object(args.bucket, f"{args.prefix}{str(current_build.id)}") + # Prune through images that are not mentioned in images-keep case "images": + images_to_keep = policy.get(stream, {}).get("images-keep", []) + prune_images(s3_client, current_build, images_to_keep, args.dry_run, bucket, prefix) + # Fully prune releases that are very old including deleting the directory in s3 for that build. + case "build": raise NotImplementedError - build.setdefault("policy-cleanup", []).append("cloud-uploads") + # Update policy-cleanup after processing all arches for the build + match action: + case "cloud-uploads": + policy_cleanup = build.setdefault("policy-cleanup", {}) + if "cloud-uploads" not in policy_cleanup: + policy_cleanup["cloud-uploads"] = True + case "images": + policy_cleanup = build.setdefault("policy-cleanup", {}) + if "images" not in policy_cleanup: + policy_cleanup["images"] = True + policy_cleanup["images-kept"] = images_to_keep + # Save the updated builds.json to local builds/builds.json save_builds_json(builds_json_data, BUILDFILES['list']) @@ -320,5 +339,39 @@ def delete_gcp_image(build, cloud_config, dry_run): return errors +def prune_images(s3, build, images_to_keep, dry_run, bucket, prefix): + images_from_meta_json = build.meta_json.get("images", []) + # Get the image names and paths currently in meta.json + current_images_data = [(name, data.get("path")) for name, data in images_from_meta_json.items()] + # Extract names to compare with images_to_keep + current_images = [name for name, _ in current_images_data] + errors = [] + + if images_to_keep == current_images: + # if images_to_keep and current_images are the same, we don't need to prune anything so conitnue. + print(f"No images to be pruned for {build.id}") + return + + for name, path in current_images_data: + if name and name not in images_to_keep: + image_prefix = os.path.join(prefix, f"{build.id}/{build.arch}/{path}") + if dry_run: + print(f"Would prune {bucket}/{image_prefix}") + else: + try: + s3.delete_object(Bucket=bucket, Key=image_prefix) + print(f"Pruned {name}") + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'NoSuchKey': + print(f"{bucket}/{image_prefix} already pruned.") + else: + errors.append(e) + if errors: + print(f"Found errors when pruning images for {build.id}:") + for e in errors: + print(e) + raise Exception("Some errors were encountered") + + if __name__ == "__main__": main()