enhance snapshot replicator (#39)

snapshot-replicator/functions/remove_snapshots.py

@@ -17,7 +17,7 @@ def deleteSnapshots(region):
         for instance in instances.split(','):
             rds = boto3.client('rds', region_name=region)
             paginator = rds.get_paginator('describe_db_snapshots')
-            page_iterator = paginator.paginate(DBInstanceIdentifier=instance)
+            page_iterator = paginator.paginate(DBInstanceIdentifier=instance, SnapshotType='manual')
             snapshots = []
             for page in page_iterator:
                  snapshots.extend(page['DBSnapshots'])

snapshot-replicator/functions/shipper.py

@@ -1,43 +1,25 @@
 import boto3
 import botocore
-import datetime
 import re
 import os
 
 source_region = os.environ['SOURCE_REGION']
 target_region = os.environ['TARGET_REGION']
 kms_key_id = os.environ['KMS_KEY_ID'] 
-iam = boto3.client('iam')  
 instances = os.environ['DB_INSTANCES']  
 
 print('Loading function')
 
-def byTimestamp(snap):  
-    if 'SnapshotCreateTime' in snap:
-        return datetime.datetime.isoformat(snap['SnapshotCreateTime'])
-    else:
-        return datetime.datetime.isoformat(datetime.datetime.now())
-
 def lambda_handler(event, context): 
-    if("Finished" in event['Records'][0]['Sns']['Message']):
-        account_ids = []
-        try:
-            iam.get_user()
-        except Exception as e:
-            account_ids.append(re.search(r'(arn:aws:sts::)([0-9]+)', str(e)).groups()[1])
-            account = account_ids[0]
-
+    if("Manual snapshot created" in event['Records'][0]['Sns']['Message']):
         source = boto3.client('rds', region_name=source_region)
-
-        for instance in instances.split(','):
-            source_instances = source.describe_db_instances(DBInstanceIdentifier=instance)
-            source_snaps = source.describe_db_snapshots(DBInstanceIdentifier=instance)['DBSnapshots']
-            source_snap = sorted(source_snaps, key=byTimestamp, reverse=True)[0]['DBSnapshotIdentifier']
-            source_snap_arn = 'arn:aws:rds:%s:%s:snapshot:%s' % (source_region, account, source_snap)
+        source_snap = event['Records'][0]['Sns']['Source']
+        snapshot_details = source.describe_db_snapshots(DBSnapshotIdentifier=source_snap)['DBSnapshots'][0]
+        if snapshot_detailts['DBInstanceIdentifier'] in instances.split(','):
+            source_snap_arn = snapshot_detailts['DBSnapshotArn'])
             target_snap_id = (re.sub('rds:', '', source_snap))
-            print('Will Copy %s to %s' % (source_snap_arn, target_snap_id))
             target = boto3.client('rds', region_name=target_region)
-
+            print('Will Copy %s to %s' % (source_snap_arn, target_snap_id))
             try:
                 response = target.copy_db_snapshot(
                 SourceDBSnapshotIdentifier=source_snap_arn,
@@ -48,5 +30,4 @@ def lambda_handler(event, context):
                 print(response)
             except botocore.exceptions.ClientError as e:
                 raise Exception("Could not issue copy command: %s" % e)
-            copied_snaps = target.describe_db_snapshots(SnapshotType='manual', DBInstanceIdentifier=instance)['DBSnapshots']
-
+

snapshot-replicator/main.tf

@@ -65,7 +65,7 @@ resource "aws_iam_role_policy_attachment" "attach_lambda_copy_policy_to_role" {
 resource "aws_iam_role_policy_attachment" "lambda_exec_role" {
   count      = var.enable ? 1 : 0
   role       = aws_iam_role.iam_for_lambda[0].name
-  policy_arn = "arn:aws:iam::aws:policy/AWSLambdaBasicExecutionRole"
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
 }
 
 resource "aws_iam_policy" "rds_lambda_create_snapshot" {
@@ -232,11 +232,10 @@ resource "aws_db_event_subscription" "default" {
   name      = "rds-manual-snapshot-${var.environment}"
   sns_topic = aws_sns_topic.rds_backup_events[0].arn
 
-  source_type = "db-instance"
-  source_ids  = var.db_instances
+  source_type = "snapshots"
 
   event_categories = [
-    "backup",
+    "creation",
   ]
 }
 

snapshot-replicator/monitoring.tf

@@ -1,3 +1,8 @@
+locals {
+  cw_alarm_custom_period = 3600 * var.custom_snapshot_rate
+  cw_alarm_daily_period  = 3600 * 24
+}
+
 resource "aws_cloudwatch_metric_alarm" "lambda_rds_snapshot_copy_errors" {
   count               = var.enable ? 1 : 0
   alarm_name          = "rds_snapshot_copy_invocation_${var.environment}_errors"
@@ -8,7 +13,7 @@ resource "aws_cloudwatch_metric_alarm" "lambda_rds_snapshot_copy_errors" {
   comparison_operator = "GreaterThanThreshold"
   threshold           = 1
   evaluation_periods  = 1
-  period              = 21600 # 6 hours
+  period              = local.cw_alarm_custom_period
 
   alarm_actions = [var.sns_topic_arn]
   ok_actions    = [var.sns_topic_arn]
@@ -28,7 +33,7 @@ resource "aws_cloudwatch_metric_alarm" "lambda_rds_snapshot_create_errors" {
   comparison_operator = "GreaterThanThreshold"
   threshold           = 1
   evaluation_periods  = 1
-  period              = 21600 # 6 hours
+  period              = local.cw_alarm_custom_period
 
   alarm_actions = [var.sns_topic_arn]
   ok_actions    = [var.sns_topic_arn]
@@ -48,7 +53,7 @@ resource "aws_cloudwatch_metric_alarm" "lambda_rds_snapshot_cleanup_errors" {
   comparison_operator = "GreaterThanThreshold"
   threshold           = 1
   evaluation_periods  = 1
-  period              = 86400 # 24 hours
+  period              = local.cw_alarm_daily_period
 
   alarm_actions = [var.sns_topic_arn]
   ok_actions    = [var.sns_topic_arn]
@@ -68,7 +73,7 @@ resource "aws_cloudwatch_metric_alarm" "invoke_rds_snapshot_lambda" {
   comparison_operator = "GreaterThanThreshold"
   threshold           = 1
   evaluation_periods  = 1
-  period              = 21600 # 6 hours
+  period              = local.cw_alarm_custom_period
 
   alarm_actions = [var.sns_topic_arn]
   ok_actions    = [var.sns_topic_arn]
@@ -88,7 +93,7 @@ resource "aws_cloudwatch_metric_alarm" "invoke_rds_cleanup_lambda" {
   comparison_operator = "GreaterThanThreshold"
   threshold           = 1
   evaluation_periods  = 1
-  period              = 86400 # 24 hours
+  period              = local.cw_alarm_daily_period
 
   alarm_actions = [var.sns_topic_arn]
   ok_actions    = [var.sns_topic_arn]