Restore testing update pr 2.x

roles/aws/aws_backup/tasks/backup_plan.yml

@@ -41,7 +41,24 @@
   register: _backup_plan_created_info
   when: _backup_plan_created is defined
 
+- name: Check if the restore testing plan exists.
+  ansible.builtin.command: >
+    aws backup list-restore-testing-plans --region {{ _aws_region }} --query "RestoreTestingPlans[?RestoreTestingPlanName=='{{ plan.name | replace("-", "_") }}'] | [0]" --output json
+  register: _testing_plan_exists
+
+- name: Create restore testing plan if it doesn't exist.
+  ansible.builtin.command: >
+    aws backup create-restore-testing-plan --restore-testing-plan "RestoreTestingPlanName={{ plan.name | replace('-', '_') }},RecoveryPointSelection={Algorithm=LATEST_WITHIN_WINDOW,RecoveryPointTypes=[\"SNAPSHOT\"],IncludeVaults=[\"{{ _vault_info.stdout | from_json | json_query('BackupVaultArn') }}\"]},ScheduleExpression=\"cron(0 0 ? * SUN *)\"" --region {{ _aws_region }}
+  register: _testing_plan_created
+  when: _testing_plan_exists.stdout == "null"
+
+- name: Get info about newly created restore testing plan.
+  ansible.builtin.command: >
+    aws backup list-restore-testing-plans --region {{ _aws_region }} --query "RestoreTestingPlans[?RestoreTestingPlanName=='{{ plan.name | replace("-", "_") }}'] | [0]"
+  register: _testing_plan_created_info
+  when: _testing_plan_created is defined
+
 - name: Tidy up and remove the yaml input file.
   ansible.builtin.file:
     path: "/tmp/aws_backup/{{ _infra_name }}/{{ plan.name }}-backup-yaml-input.yml"
-    state: absent
+    state: absent

roles/aws/aws_backup/tasks/main.yml

@@ -28,6 +28,11 @@
     - aws_backup.vault.encryption_key != "Default"
     - _vault_exists.stdout == "null"
 
+- name: Get the backup vault info.
+  ansible.builtin.command: >
+    aws backup list-backup-vaults --region {{ _aws_region }} --query "BackupVaultList[?BackupVaultName=='{{ aws_backup.vault.name }}'] | [0]" --output json
+  register: _vault_info
+
 - name: Create off-site backup vault without user-provided KMS key.
   ansible.builtin.command: >
     aws backup create-backup-vault --backup-vault-name {{ aws_backup.copy_vault.name }} --region {{ aws_backup.copy_vault.region }}
@@ -60,3 +65,7 @@
   loop_control:
     loop_var: plan
   when: aws_backup.plans | length
+
+- name: Include backup validation role.
+  ansible.builtin.include_role:
+    name: aws/aws_backup_validation

roles/aws/aws_backup/tasks/resource.yml

@@ -4,6 +4,11 @@
     aws backup list-backup-plans --region {{ _aws_region }} --query "BackupPlansList[?BackupPlanName=='{{ backup.backup_plan_name }}'] | [0]" --output json
   register: _backup_plan_info
 
+- name: Get restore testing plan info.
+  ansible.builtin.command: >
+    aws backup list-restore-testing-plans --region {{ _aws_region }} --query "RestoreTestingPlans[?RestoreTestingPlanName=='{{ backup.backup_plan_name | replace("-", "_") }}'] | [0]" --output json
+  register: _testing_plan_info
+
 - name: Gather AWS account ID if it isn't already set.
   amazon.aws.aws_caller_info:
     profile: "{{ _aws_profile }}"
@@ -59,3 +64,14 @@
   ansible.builtin.command: >
     aws backup create-backup-selection --backup-plan-id {{ _backup_plan_info.stdout | from_json | json_query('BackupPlanId') }} --backup-selection "{\"SelectionName\":\"{{ backup.selection_name }}\",\"IamRoleArn\":\"{{ _iam_role_arn }}\",\"Resources\":[\"{{ _resource_arn }}\"]}" --region {{ _aws_region }}
   when: _selection_exists.stdout | length == 0
+
+- name: Check if the restore testing selection exists.
+  ansible.builtin.command: >
+    aws backup list-restore-testing-selections --restore-testing-plan-name {{ _testing_plan_info.stdout | from_json | json_query('RestoreTestingPlanName') }} --query "RestoreTestingSelections[?RestoreTestingSelectionName=='{{ backup.selection_name | replace('-', '_') }}'].RestoreTestingSelectionName" --output text --region {{ _aws_region }}
+  register: _testing_selection_exists
+  when: _testing_plan_info.stdout != "null"
+
+- name: Assign resource to AWS restore testing plan.
+  ansible.builtin.command: >
+    aws backup create-restore-testing-selection --restore-testing-plan-name {{ _testing_plan_info.stdout | from_json | json_query('RestoreTestingPlanName') }} --restore-testing-selection "{\"RestoreTestingSelectionName\":\"{{ backup.selection_name | replace('-', '_') }}\",\"IamRoleArn\":\"{{ _iam_role_arn }}\",\"ProtectedResourceArns\":[\"{{ _resource_arn }}\"],\"ProtectedResourceType\":\"{{ backup.selection_name.split('-')[0] }}\",\"ValidationWindowHours\":1}" --region {{ _aws_region }}
+  when: _testing_plan_info.stdout != "null" and _testing_selection_exists.stdout | length == 0

roles/aws/aws_backup_validation/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+aws_backup_validation:
+  name: 'RestoreValidation'
+  description: 'Restore validation for'
+  timeout: 60
+  runtime: python3.12
+  handler: "lambda_handler"
+  resources:
+    - EC2
+    - RDS
+    #- EFS

roles/aws/aws_backup_validation/files/pass_role_backup.j2

@@ -0,0 +1,12 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "backup.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

roles/aws/aws_backup_validation/files/trusted_entitites.j2

@@ -0,0 +1,12 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "lambda.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}

roles/aws/aws_backup_validation/tasks/main.yml

@@ -0,0 +1,105 @@
+---
+- name: Create a role and attach policies
+  amazon.aws.iam_role:
+    name: LambdaBackupRestoreRole
+    assume_role_policy_document: "{{ lookup('file','trusted_entitites.j2') }}"
+    managed_policies:
+      - arn:aws:iam::aws:policy/AmazonEC2FullAccess
+      - arn:aws:iam::aws:policy/service-role/AmazonEFSCSIDriverPolicy
+      - arn:aws:iam::aws:policy/AmazonRDSFullAccess
+      - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess
+  register: _created_iam_lambda_role
+
+- name: Create an IAM Managed Policy for passing roles
+  amazon.aws.iam_managed_policy:
+    policy_name: "PassRole"
+    policy:
+      Version: "2012-10-17"
+      Statement:
+        - Effect: "Allow"
+          Action: "iam:PassRole"
+          Resource: "*"
+    state: present
+  register: _pass_role
+
+- name: Update AWSBackupDefaultServiceRole
+  amazon.aws.iam_role:
+    name: AWSBackupDefaultServiceRole
+    assume_role_policy_document: "{{ lookup('file','pass_role_backup.j2') }}"
+    managed_policies:
+      - arn:aws:iam::aws:policy/service-role/AWSBackupServiceRolePolicyForBackup
+      - arn:aws:iam::aws:policy/service-role/AWSBackupServiceRolePolicyForRestores
+      - "{{ _pass_role.policy.arn }}"
+
+- name: Sleep for 20 seconds for IAM before Lambda creation
+  ansible.builtin.wait_for:
+    timeout: 20
+
+- name: Clean and set python functions
+  block:
+    - name: Check and clean any previous Lambda files
+      ansible.builtin.file:
+        path: "{{ _ce_provision_build_dir }}/{{ item }}_validation.py"
+        state: absent
+      loop: "{{ aws_backup_validation.resources }}"
+
+    - name: Write Lambda functions
+      ansible.builtin.template:
+        src: "{{ item }}_validation.py.j2"
+        dest: "{{ _ce_provision_build_dir }}/{{ item }}_validation.py"
+      loop: "{{ aws_backup_validation.resources }}"
+
+    - name: Create a zip archive of Lambda functions
+      community.general.archive:
+        path: "{{ _ce_provision_build_dir }}/{{ item }}_validation.py"
+        dest: "{{ _ce_provision_build_dir }}/{{ item }}_validation.zip"
+        format: zip
+      loop: "{{ aws_backup_validation.resources }}"
+
+- name: Create Lambda functions
+  amazon.aws.lambda:
+    name: "{{ aws_backup_validation.name }}_{{ item }}"
+    description: "{{ aws_backup_validation.description }} {{ item }} new comment to update function"
+    region: "{{ _aws_region }}"
+    timeout: "{{ aws_backup_validation.timeout }}"
+    zip_file: "{{ _ce_provision_build_dir }}/{{ item }}_validation.zip"
+    state: present
+    runtime: "{{ aws_backup_validation.runtime }}"
+    role: "{{ _created_iam_lambda_role.iam_role.arn }}"
+    handler: "{{ item }}_validation.{{ aws_backup_validation.handler }}"
+    tags:
+      Name: "{{ item }}_backup_validation"
+  register: _lambda_functions
+  loop: "{{ aws_backup_validation.resources }}"
+
+- name: Remove non UTF-8 item
+  ansible.builtin.set_fact:
+    _lambda_functions: "{{ _lambda_functions | ansible.utils.remove_keys(target=['ZipFile', 'location', 'item.invocation']) }}"
+
+- name: Create EventBridge
+  amazon.aws.cloudwatchevent_rule:
+    name: "{{ item.configuration.function_name }}"
+    description: "{{ item.configuration.description }}"
+    state: present
+    region: "{{ _aws_region }}"
+    event_pattern: '{ "source": ["aws.backup"], "detail-type": ["Restore Job State Change"], "detail": { "resourceType": ["{{ item.item }}"], "status": ["COMPLETED"] } }'
+    targets:
+      - id: "{{ item.configuration.function_name }}"
+        arn: "{{ (item.configuration.function_arn.split(':') | map('trim'))[:-1] | join(':') }}" # Remove the version number from ARN
+  register: _event_bridges
+  loop: "{{ _lambda_functions.results }}"
+
+- name: Generate unique string
+  ansible.builtin.set_fact:
+    _rand_str: "{{ lookup('community.general.random_string', length=8, special=false, min_lower=2, min_numeric=2, min_upper=2) }}"
+
+- name: Update Lambda policy
+  amazon.aws.lambda_policy:
+    state: present
+    function_name: "{{ item.item.configuration.function_name }}"
+    statement_id: "{{ item.item.configuration.function_name }}_{{ _rand_str }}"
+    action: lambda:InvokeFunction
+    principal: events.amazonaws.com
+    source_arn: "{{ item.rule.arn }}"
+    region: "{{ _aws_region }}"
+  loop: "{{ _event_bridges.results }}"

roles/aws/aws_backup_validation/templates/EC2_validation.py.j2

@@ -0,0 +1,144 @@
+import json
+import boto3
+import socket
+import time
+
+# Defining Clients
+#s3_cli = boto3.client('s3', region_name='eu-west-2')
+backup_cli = boto3.client('backup', region_name="{{ _aws_region }}")
+ec2_cli = boto3.client("ec2", region_name="{{ _aws_region }}")
+ssm_cli = boto3.client('ssm', region_name="{{ _aws_region }}")
+ses_cli = boto3.client('ses', region_name="{{ _aws_region }}")
+
+# Debugger
+#boto3.set_stream_logger('')
+
+def lambda_handler(event, context):
+
+  mail_title = ""
+  mail_body = ""
+
+  print("Gathering instance id.")
+  ec2_instance_id=event['detail']['createdResourceArn'].split("/",1)[1]
+
+  print("Gathering instance details.")
+  ec2_instances=ec2_cli.describe_instances()
+
+  instance_exist = False
+
+  for reservation in ec2_instances["Reservations"]:
+    for instance in reservation["Instances"]:
+      if ec2_instance_id == instance["InstanceId"]:
+        tags = instance['Tags']
+        instance_type = instance["InstanceType"]
+        private_ip = instance["PrivateIpAddress"]
+        mail_body = mail_body + "Instance is restored!\n"
+        instance_exist = True
+
+  if instance_exist:
+    port = 22
+
+    print("Gathering instance name.")
+    instance_name = ''
+    for tag in tags:
+      if tag['Key'] == 'Name':
+        instance_name = tag['Value']
+
+    print("Testing connection!")
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.settimeout(5.0)
+    result = sock.connect_ex((private_ip,port))
+
+    print(result)
+
+    # If result was not success
+    if result != 0:
+      mail_body = mail_body + "Connection on " + str(port) + " is not working, this could be caused by firewall not accepting connections.\n"
+    else:
+      mail_body = mail_body + "Connection on " + str(port) + " is working!\n"
+
+    # Check if SSM is set up for instance
+    ssm_status = ssm_cli.get_connection_status(Target=ec2_instance_id)
+
+    if ssm_status['Status'] == 'connected':
+      # Run scripts on EC2
+      print("Creating script!")
+      script = """
+      echo "Server info:"
+      hostnamectl
+      echo "Disk usage:"
+      df -h
+      """
+
+      print("Running command!")
+      command_response = ssm_cli.send_command(
+        DocumentName ='AWS-RunShellScript',
+        Parameters = {'commands': [script]},
+        InstanceIds = [
+          ec2_instance_id
+        ]
+      )
+
+      print("Gathering commands details!")
+      time.sleep(10)
+      c_res = ssm_cli.get_command_invocation(
+        CommandId=command_response['Command']['CommandId'],
+        InstanceId=ec2_instance_id
+      )
+
+      print(c_res['StandardOutputContent'])
+      mail_title = "Success: " + instance_name
+      instance_message = "Instance " + instance_name + " - " + ec2_instance_id + " was restored.\n"
+      mail_body = mail_body + "SSM is working and these are the details of the instance:\n" + c_res['StandardOutputContent']
+
+    else:
+      mail_title = "Warning: " + instance_name
+      mail_body = mail_body + "SSM is not configured or accessible!\n"
+
+    print("Validating Restore job!")
+    backup_cli.put_restore_validation_result(
+      RestoreJobId=event['detail']['restoreJobId'],
+      ValidationStatus="SUCCESSFUL",
+      ValidationStatusMessage=""
+    )
+
+  else:
+    print("Validating Restore job!")
+    backup_cli.put_restore_validation_result(
+      RestoreJobId=event['detail']['restoreJobId'],
+      ValidationStatus="FAILED",
+      ValidationStatusMessage=""
+    )
+
+    mail_title = "Failed!"
+    mail_body = mail_body + "Instance " + ec2_instance_id + " is not running!"
+
+  print("Sending email!")
+  response = ses_cli.send_email(
+  Destination={
+    'BccAddresses': [
+    ],
+    'CcAddresses': [],
+    'ToAddresses': [
+      'sysadm@codeenigma.com'
+    ],
+  },
+  Message={
+    'Body': {
+      'Text': {
+        'Charset': 'UTF-8',
+        'Data': mail_body,
+      },
+    },
+    'Subject': {
+      'Charset': 'UTF-8',
+      'Data': 'Lambda Backup validation: ' + mail_title,
+    },
+  },
+  Source='Lambda Backup Validation <sysadm@codeenigma.com>',
+  )
+
+  return {
+    'statusCode': 200,
+    'body':  event
+  }