From 4c53840013f2f2a9f8c191aa854ebe2a1f6c62d2 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Thu, 11 Jul 2024 17:32:34 +0100 Subject: [PATCH 1/9] Add python script and requirements to automatically create daily backups of gcp filestores --- .../gcp-filestore-backups.py | 134 ++++++++++++++++++ .../gcp-filestore-backups/requirements.txt | 1 + 2 files changed, 135 insertions(+) create mode 100644 helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py create mode 100644 helm-charts/images/gcp-filestore-backups/requirements.txt diff --git a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py new file mode 100644 index 000000000..65b1184d8 --- /dev/null +++ b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py @@ -0,0 +1,134 @@ +import argparse +import json +import subprocess +import time +from datetime import datetime, timedelta + +import jmespath + + +def main(args): + # Get a JSON object of the filestore backups. We filter by project and region + # in the gcloud command. + filestore_backups = subprocess.check_output( + [ + "gcloud", + "filestore", + "backups", + "list", + "--format=json", + f"--project={args.project}", + f"--region={args.region}", + ], + text=True, + ) + filestore_backups = json.loads(filestore_backups) + + # Filter returned backups by filestore and share names + filestore_backups = jmespath.search( + f"[?sourceFileShare == '{args.filestore_share_name}' && contains(sourceInstance, '{args.filestore_name}')]", + filestore_backups, + ) + + # Parse `createTime` to a datetime object for comparison + filestore_backups = [ + { + k: ( + datetime.strptime(v.split(".")[0], "%Y-%m-%dT%H:%M:%S") + if k == "createTime" + else v + ) + for k, v in backup.items() + } + for backup in filestore_backups + ] + + # Generate a list of filestore backups that are less than 24 hours old + recent_filestore_backups = [ + backup + for backup in filestore_backups + if datetime.now() - backup["createTime"] < timedelta(days=1) + ] + + # Generate a list of filestore backups that are older than our set retention period + old_filestore_backups = [ + backup + for backup in filestore_backups + if datetime.now() - backup["createTime"] > timedelta(days=args.retention_days) + ] + + if len(recent_filestore_backups) == 0: + print( + f"There have been no recent backups of the filestore for project {args.project}. Creating a backup now..." + ) + + subprocess.check_call( + [ + "gcloud", + "filestore", + "backups", + "create", + f"{args.filestore_name}-{args.filestore_share_name}-backup-{datetime.now().strftime('%Y-%m-%d')}", + f"--file-share={args.filestore_share_name}", + f"--instance={args.filestore_name}", + f"--instance-location={args.region}-b", + f"--region={args.region}", + # This operation can take a long time to complete and will only take + # longer as filestores grow, hence we use the `--async` flag to + # return immediately, without waiting for the operation in progress + # to complete. Given that we only expect to be creating a backup + # once a day, this feels safe enough to try for now. + "--async", + ] + ) + else: + print("Recent backup found.") + + if len(old_filestore_backups) > 0: + print( + f"Filestore backups older than {args.retention_days} days have been found. They will be deleted." + ) + + for backup in old_filestore_backups: + subprocess.check_call( + [ + "gcloud", + "filestore", + "backups", + "delete", + backup["name"].split("/")[-1], + f"--region={args.region}", + "--quiet", # Otherwise we are prompted to confirm deletion + ] + ) + else: + print("No outdated backups found.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="") + + parser.add_argument( + "project", + type=str, + help="", + ) + parser.add_argument("filestore_name", metavar="filestore-name", type=str, help="") + parser.add_argument( + "region", + type=str, + help="", + ) + parser.add_argument( + "--filestore-share-name", + type=str, + default="homes", + help="", + ) + parser.add_argument("--retention-days", type=int, default=5, help="") + + args = parser.parse_args() + + while True: + main(args) + time.sleep(600) diff --git a/helm-charts/images/gcp-filestore-backups/requirements.txt b/helm-charts/images/gcp-filestore-backups/requirements.txt new file mode 100644 index 000000000..45c1e038e --- /dev/null +++ b/helm-charts/images/gcp-filestore-backups/requirements.txt @@ -0,0 +1 @@ +jmespath From 14edf78a46c395394812a50eb83826b900c72743 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Thu, 11 Jul 2024 17:42:23 +0100 Subject: [PATCH 2/9] Remove metavar argument This is a positional argument, so if it's a hyphen or an underscore is irrelevant --- .../images/gcp-filestore-backups/gcp-filestore-backups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py index 65b1184d8..64cbcfedb 100644 --- a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py +++ b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py @@ -113,7 +113,7 @@ def main(args): type=str, help="", ) - parser.add_argument("filestore_name", metavar="filestore-name", type=str, help="") + parser.add_argument("filestore_name", type=str, help="") parser.add_argument( "region", type=str, From bba1e5aac7a95c580f794b189e964a646b4178c0 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 12 Jul 2024 10:46:02 +0100 Subject: [PATCH 3/9] Add a comment --- .../images/gcp-filestore-backups/gcp-filestore-backups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py index 64cbcfedb..c1ebbb6d7 100644 --- a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py +++ b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py @@ -131,4 +131,4 @@ def main(args): while True: main(args) - time.sleep(600) + time.sleep(600) # 60 seconds * 10 for 10 minutes sleep period From 6a7ccff6e5240e7e2caad2661ffab19d130a7796 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 12 Jul 2024 14:12:18 +0100 Subject: [PATCH 4/9] Refactor into functions --- .../gcp-filestore-backups.py | 155 ++++++++++++++---- 1 file changed, 122 insertions(+), 33 deletions(-) diff --git a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py index c1ebbb6d7..dde3b098b 100644 --- a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py +++ b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py @@ -7,31 +7,49 @@ import jmespath -def main(args): - # Get a JSON object of the filestore backups. We filter by project and region - # in the gcloud command. - filestore_backups = subprocess.check_output( +def get_existing_backups( + project: str, region: str, filestore_name: str, filestore_share_name: str +): + """List existing backups of a share on a filestore using the gcloud CLI. + We filter the backups based on: + - GCP project + - GCP region + - Filestore name + - Filestore share name + + Args: + project (str): The GCP project the filestore is located in + region (str): The region the filestore is located in, e.g., us-central1 + filestore_name (str): The name of the filestore instance + filestore_share_name (str): The name of the share on the filestore instance + + Returns: + list(dict): A JSON-like object, where each dict-entry in the list describes + an existing backup of the filestore + """ + # Get all existing backups in the selected project and region + backups = subprocess.check_output( [ "gcloud", "filestore", "backups", "list", "--format=json", - f"--project={args.project}", - f"--region={args.region}", + f"--project={project}", + f"--region={region}", ], text=True, ) - filestore_backups = json.loads(filestore_backups) + backups = json.loads(backups) # Filter returned backups by filestore and share names - filestore_backups = jmespath.search( - f"[?sourceFileShare == '{args.filestore_share_name}' && contains(sourceInstance, '{args.filestore_name}')]", - filestore_backups, + backups = jmespath.search( + f"[?sourceFileShare == '{filestore_share_name}' && contains(sourceInstance, '{filestore_name}')]", + backups, ) - # Parse `createTime` to a datetime object for comparison - filestore_backups = [ + # Parse `createTime` property into a datetime object for comparison + backups = [ { k: ( datetime.strptime(v.split(".")[0], "%Y-%m-%dT%H:%M:%S") @@ -40,26 +58,76 @@ def main(args): ) for k, v in backup.items() } - for backup in filestore_backups + for backup in backups ] - # Generate a list of filestore backups that are less than 24 hours old - recent_filestore_backups = [ + return backups + + +def filter_backups_into_recent_and_old( + backups: list, retention_days: int, day_freq: int = 1 +): + """Filter the list of backups into two groups: + - Recently created backups that were created within our backup window, + defined by day_freq + - Out of date back ups that are older than our retention window, defined by + retention days + + Args: + backups (list(dict)): A JSON-like object defining the existing backups + for the filestore and share we care about + retention_days (int): The number of days above which a backup is considered + to be out of date + day_freq (int, optional): The time period in days for which we create a + backup. Defaults to 1 (ie. daily backups). + + Returns: + recent_backups (list(dict)): A JSON-like object containing all existing + backups with a `createTime` within our backup window + old_backups (list(dict)): A JSON-like object containing all existing + backups with a `createTime` older than our retention window + """ + # Generate a list of filestore backups that are younger than our backup window + recent_backups = [ backup - for backup in filestore_backups - if datetime.now() - backup["createTime"] < timedelta(days=1) + for backup in backups + if datetime.now() - backup["createTime"] < timedelta(days=day_freq) ] # Generate a list of filestore backups that are older than our set retention period - old_filestore_backups = [ + old_backups = [ backup - for backup in filestore_backups - if datetime.now() - backup["createTime"] > timedelta(days=args.retention_days) + for backup in backups + if datetime.now() - backup["createTime"] > timedelta(days=retention_days) ] + if len(old_backups) > 0: + print( + f"Filestore backups older than {retention_days} days have been found. They will be deleted." + ) - if len(recent_filestore_backups) == 0: + return recent_backups, old_backups + + +def create_backup_if_necessary( + backups: list, + filestore_name: str, + filestore_share_name: str, + project: str, + region: str, +): + """If no recent backups have been found, create a new backup using the gcloud CLI + + Args: + backups (list(dict)): A JSON-like object containing details of recently + created backups + filestore_name (str): The name of the Filestore instance to backup + filestore_share_name (str): The name of the share on the Filestore to backup + project (str): The GCP project within which to create a backup + region (str): The GCP region to create the backup in, e.g. us-central1 + """ + if len(backups) == 0: print( - f"There have been no recent backups of the filestore for project {args.project}. Creating a backup now..." + f"There have been no recent backups of the filestore for project {project}. Creating a backup now..." ) subprocess.check_call( @@ -68,11 +136,11 @@ def main(args): "filestore", "backups", "create", - f"{args.filestore_name}-{args.filestore_share_name}-backup-{datetime.now().strftime('%Y-%m-%d')}", - f"--file-share={args.filestore_share_name}", - f"--instance={args.filestore_name}", - f"--instance-location={args.region}-b", - f"--region={args.region}", + f"{filestore_name}-{filestore_share_name}-backup-{datetime.now().strftime('%Y-%m-%d')}", + f"--file-share={filestore_share_name}", + f"--instance={filestore_name}", + f"--instance-location={region}-b", + f"--region={region}", # This operation can take a long time to complete and will only take # longer as filestores grow, hence we use the `--async` flag to # return immediately, without waiting for the operation in progress @@ -84,12 +152,16 @@ def main(args): else: print("Recent backup found.") - if len(old_filestore_backups) > 0: - print( - f"Filestore backups older than {args.retention_days} days have been found. They will be deleted." - ) - for backup in old_filestore_backups: +def delete_old_backups(backups: list, region: str): + """If out of date backups exist, delete them using the gcloud CLI + + Args: + backups (list(dict)): A JSON-like object containing out of date backups + region (str): The GCP region the backups exist in, e.g. us-central1 + """ + if len(backups) > 0: + for backup in backups: subprocess.check_call( [ "gcloud", @@ -97,7 +169,7 @@ def main(args): "backups", "delete", backup["name"].split("/")[-1], - f"--region={args.region}", + f"--region={region}", "--quiet", # Otherwise we are prompted to confirm deletion ] ) @@ -105,6 +177,23 @@ def main(args): print("No outdated backups found.") +def main(args): + filestore_backups = get_existing_backups( + args.project, args.region, args.filestore_name, args.filestore_share_name + ) + recent_filestore_backups, old_filestore_backups = ( + filter_backups_into_recent_and_old(filestore_backups, args.retention_days) + ) + create_backup_if_necessary( + recent_filestore_backups, + args.project, + args.filestore_name, + args.filestore_share_name, + args.region, + ) + delete_old_backups(old_filestore_backups, args.region) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="") From 6f7a3a14cfb3a8c340f88c3366a02f82e66b5869 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 12 Jul 2024 14:16:24 +0100 Subject: [PATCH 5/9] Add zone input to parse to backup create command --- .../gcp-filestore-backups/gcp-filestore-backups.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py index dde3b098b..ad2166156 100644 --- a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py +++ b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py @@ -114,6 +114,7 @@ def create_backup_if_necessary( filestore_share_name: str, project: str, region: str, + zone: str, ): """If no recent backups have been found, create a new backup using the gcloud CLI @@ -124,6 +125,7 @@ def create_backup_if_necessary( filestore_share_name (str): The name of the share on the Filestore to backup project (str): The GCP project within which to create a backup region (str): The GCP region to create the backup in, e.g. us-central1 + zone (str): The GCP zone to create the backup in, e.g. us-central1-b """ if len(backups) == 0: print( @@ -139,7 +141,7 @@ def create_backup_if_necessary( f"{filestore_name}-{filestore_share_name}-backup-{datetime.now().strftime('%Y-%m-%d')}", f"--file-share={filestore_share_name}", f"--instance={filestore_name}", - f"--instance-location={region}-b", + f"--instance-zone={zone}", f"--region={region}", # This operation can take a long time to complete and will only take # longer as filestores grow, hence we use the `--async` flag to @@ -190,6 +192,7 @@ def main(args): args.filestore_name, args.filestore_share_name, args.region, + args.zone, ) delete_old_backups(old_filestore_backups, args.region) @@ -208,6 +211,11 @@ def main(args): type=str, help="", ) + parser.add_argument( + "zone", + type=str, + help="", + ) parser.add_argument( "--filestore-share-name", type=str, From fff45a6baf8a3abac69adea6f76a691634048835 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 12 Jul 2024 14:23:56 +0100 Subject: [PATCH 6/9] Add help strings to argparser --- .../gcp-filestore-backups.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py index ad2166156..766b3fd19 100644 --- a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py +++ b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py @@ -198,31 +198,42 @@ def main(args): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="") + parser = argparse.ArgumentParser( + description="""Uses the gcloud CLI to check for existing backups of a GCP + Filestore, creates a new backup if necessary, and deletes outdated backups + """ + ) + parser.add_argument( + "filestore_name", type=str, help="The name of the GCP Filestore to backup" + ) parser.add_argument( "project", type=str, - help="", + help="The GCP project the Filestore belongs to", ) - parser.add_argument("filestore_name", type=str, help="") parser.add_argument( "region", type=str, - help="", + help="The GCP region the Filestore is deployed in, e.g. us-central1", ) parser.add_argument( "zone", type=str, - help="", + help="The GCP zone the Filestore is deployed in, e.g. us-central1-b", ) parser.add_argument( "--filestore-share-name", type=str, default="homes", - help="", + help="The name of the share on the Filestore to backup", + ) + parser.add_argument( + "--retention-days", + type=int, + default=5, + help="The number of days to store backups for", ) - parser.add_argument("--retention-days", type=int, default=5, help="") args = parser.parse_args() From a472d68df0698d553f5e5e747d0df1dd9cbfc746 Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 12 Jul 2024 14:28:16 +0100 Subject: [PATCH 7/9] Generate the GCP region from the zone --- .../gcp-filestore-backups.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py index 766b3fd19..4957f32ba 100644 --- a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py +++ b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py @@ -7,6 +7,13 @@ import jmespath +def extract_region_from_zone(zone: str): + """ + Parse a GCP zone (e.g. us-central1-b) to return a region (e.g. us-central1) + """ + return "-".join(zone.split("-"[:2])) + + def get_existing_backups( project: str, region: str, filestore_name: str, filestore_share_name: str ): @@ -180,8 +187,9 @@ def delete_old_backups(backups: list, region: str): def main(args): + region = extract_region_from_zone(args.zone) filestore_backups = get_existing_backups( - args.project, args.region, args.filestore_name, args.filestore_share_name + args.project, region, args.filestore_name, args.filestore_share_name ) recent_filestore_backups, old_filestore_backups = ( filter_backups_into_recent_and_old(filestore_backups, args.retention_days) @@ -191,10 +199,10 @@ def main(args): args.project, args.filestore_name, args.filestore_share_name, - args.region, + region, args.zone, ) - delete_old_backups(old_filestore_backups, args.region) + delete_old_backups(old_filestore_backups, region) if __name__ == "__main__": @@ -212,11 +220,6 @@ def main(args): type=str, help="The GCP project the Filestore belongs to", ) - parser.add_argument( - "region", - type=str, - help="The GCP region the Filestore is deployed in, e.g. us-central1", - ) parser.add_argument( "zone", type=str, From 87139cc2a796dc56277ca627fa41cb63bf18281b Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 12 Jul 2024 14:54:49 +0100 Subject: [PATCH 8/9] Fix typos introduced in refactor --- .../images/gcp-filestore-backups/gcp-filestore-backups.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py index 4957f32ba..ba7869afb 100644 --- a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py +++ b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py @@ -11,7 +11,7 @@ def extract_region_from_zone(zone: str): """ Parse a GCP zone (e.g. us-central1-b) to return a region (e.g. us-central1) """ - return "-".join(zone.split("-"[:2])) + return "-".join(zone.split("-")[:2]) def get_existing_backups( @@ -196,9 +196,9 @@ def main(args): ) create_backup_if_necessary( recent_filestore_backups, - args.project, args.filestore_name, args.filestore_share_name, + args.project, region, args.zone, ) From 3fa1b2f6a236b3f561b45c60ca6016d0340dd7ed Mon Sep 17 00:00:00 2001 From: Sarah Gibson Date: Fri, 12 Jul 2024 14:55:04 +0100 Subject: [PATCH 9/9] Update comment about async flag --- .../images/gcp-filestore-backups/gcp-filestore-backups.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py index ba7869afb..59be5238e 100644 --- a/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py +++ b/helm-charts/images/gcp-filestore-backups/gcp-filestore-backups.py @@ -155,6 +155,10 @@ def create_backup_if_necessary( # return immediately, without waiting for the operation in progress # to complete. Given that we only expect to be creating a backup # once a day, this feels safe enough to try for now. + # The `gcloud filestore backups list` command is instantaneously + # populated with new backups, even if they are not done creating. + # So we don't have to worry about the async flag not taking those + # into account. "--async", ] )