Feature/velero

playbooks/dns_playbook.yaml

@@ -0,0 +1,16 @@
+#
+# Ran to repaire DNS in case of lost Proxy Service. (For example after a Recovery Playbook is ran)
+#
+---
+- hosts: localhost
+ connection: local
+ gather_facts: no
+ tasks:
+ - include_tasks: tasks/pre-flight.yaml
+
+ - name: Kong DNS
+ include_role:
+ name: ms3_inc.tavros.kong_dns
+ vars:
+ cluster_state: present
+ tags: [ kong_dns ]

playbooks/provision_playbook.yaml

@@ -27,12 +27,24 @@
  include_role:
  name: ms3_inc.tavros.fluxtoolkit
  tags: [ fluxtoolkit ]
-
+ 
  - name: sealed-secrets
  include_role:
  name: ms3_inc.tavros.sealed_secrets
  tags: [ sealed_secrets ]
 
+ - name: cert-manager
+ include_role:
+ name: ms3_inc.tavros.cert_manager
+ when: cert_manager.enabled
+ tags: [ cert_manager ]
+
+ - name: Velero
+ include_role:
+ name: ms3_inc.tavros.velero
+ when: velero.enabled
+ tags: [ velero ]
+
  - name: PostgreSQL
  include_role:
  name: ms3_inc.tavros.postgresql
@@ -44,12 +56,6 @@
  when: kuma.enabled
  tags: [ kuma ]
 
- - name: cert-manager
- include_role:
- name: ms3_inc.tavros.cert_manager
- when: cert_manager.enabled
- tags: [ cert_manager ]
-
  - name: Kong
  include_role:
  name: ms3_inc.tavros.kong
@@ -142,6 +148,7 @@
  - 'gitea'
  - 'jaeger'
  - 'jenkins'
+ - 'velero'
 
  - include_role:
  name: ms3_inc.tavros.kops

playbooks/provision_playbook/default_vars.yaml

@@ -84,3 +84,5 @@ all:
  keycloak:
  realm: 'prod'
  flux: {}
+ velero:
+ enabled: false

playbooks/recovery_playbook.yaml

@@ -0,0 +1,48 @@
+#
+# To run this we assume you have Verlaro enabled for AWS, or AKS. On AWS Cert Manager is also required for the Kops snapshotController.
+#
+---
+- hosts: localhost
+ connection: local
+ gather_facts: no
+ tasks:
+ - include_tasks: tasks/pre-flight.yaml
+
+ - name: kOps
+ include_role:
+ name: ms3_inc.tavros.kops
+ when: kops.enabled
+ vars:
+ cluster_state: present
+ tags: [ kops ]
+
+ - name: aks
+ include_role:
+ name: ms3_inc.tavros.aks
+ when: aks.enabled
+ vars:
+ cluster_state: present
+ tags: [ aks ]
+
+ - name: Install and Configure Components
+ block:
+ - name: Flux GitOps Toolkit
+ include_role:
+ name: ms3_inc.tavros.fluxtoolkit
+ tags: [ fluxtoolkit ]
+
+ - name: sealed-secrets
+ include_role:
+ name: ms3_inc.tavros.sealed_secrets
+ tags: [ sealed_secrets ]
+
+ - name: cert-manager
+ include_role:
+ name: ms3_inc.tavros.cert_manager
+ when: kubernetes_cluster.cloud_provider == 'aws' 
+ tags: [ cert_manager ]
+
+ - name: Velero
+ include_role:
+ name: ms3_inc.tavros.velero
+ tags: [ velero ]

roles/kops/tasks/create_cluster.yaml

@@ -16,6 +16,35 @@
  block_public_policy: true
  restrict_public_buckets: true
 
+- name: Template SP Policy
+ template:
+ src: aws-sp-policy.j2
+ dest: /tmp/kops/aws-sp-policy.json
+
+- name: Create AWS SP User
+ shell: |
+ aws iam create-user --user-name {{kops.state_bucket}}-sp
+ register: result
+
+- name: Create SP Policy
+ shell: |
+ aws iam put-user-policy \
+ --user-name {{kops.state_bucket}}-sp \
+ --policy-name {{kops.state_bucket}}-sp \
+ --policy-document file:///tmp/kops/aws-sp-policy.json
+ register: result
+
+- name: Create Access Key
+ shell: |
+ aws iam create-access-key --user-name {{kops.state_bucket}}-sp
+ register: result
+
+- name: Merge User Creds
+ set_fact:
+ kops: "{{ kops | combine({ 'sp': { 'client_id': creds.AccessKey.AccessKeyId, 'client_secret': creds.AccessKey.SecretAccessKey } } , recursive=true) }}"
+ vars:
+ creds: "{{ result.stdout | from_json }}"
+
 - name: Generate an OpenSSH Key Pair
  community.crypto.openssh_keypair:
  path: /tmp/kops/id_rsa
@@ -71,6 +100,11 @@
  cloudConfig:
  awsEBSCSIDriver:
  enabled: true
+ snapshotController:
+ enabled: true
+ certManager:
+ enabled: true
+ managed: false
  metricsServer:
  enabled: true
  insecure: true
@@ -98,10 +132,10 @@
 
  exit 101
 
-- name: Wait for DNS zone propagation
- when: (cluster_state == 'present') and ('dry-run' not in ansible_run_tags)
- wait_for:
- host: api.{{ cluster_fqdn }}
+# - name: Wait for DNS zone propagation
+#  when: (cluster_state == 'present') and ('dry-run' not in ansible_run_tags)
+#  wait_for:
+#  host: api.{{ cluster_fqdn }}
 
 - name: Export Kube Context
  environment:

roles/kops/tasks/delete_cluster.yaml

@@ -18,7 +18,25 @@
  echo "Failed to delete cluster"
  exit 1
 
-- name: Delete kOps S3 Bucket
- amazon.aws.aws_s3:
- bucket: "{{ kops.state_bucket }}"
- mode: delete
+- name: Get SP Access Keys
+ shell: |
+ aws iam list-access-keys --user-name {{ kops.state_bucket }}-sp
+ register: result
+
+- name: Delete SP Access Key
+ shell: |
+ aws iam delete-access-key --user-name {{ kops.state_bucket }}-sp --access-key-id {{ item.AccessKeyId }}
+ loop: "{{ (result.stdout | from_json).AccessKeyMetadata }}"
+ register: result
+
+- name: Delete SP Policy
+ shell: |
+ aws iam delete-user-policy \
+ --user-name {{ kops.state_bucket }}-sp \
+ --policy-name {{ kops.state_bucket }}-sp
+ register: result
+
+- name: Delete AWS SP User
+ shell: |
+ aws iam delete-user --user-name {{ kops.state_bucket }}-sp
+ register: result

roles/kops/templates/aws-sp-policy.j2

@@ -0,0 +1,39 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "ec2:DescribeVolumes",
+ "ec2:DescribeSnapshots",
+ "ec2:CreateTags",
+ "ec2:CreateVolume",
+ "ec2:CreateSnapshot",
+ "ec2:DeleteSnapshot"
+ ],
+ "Resource": "*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:GetObject",
+ "s3:DeleteObject",
+ "s3:PutObject",
+ "s3:AbortMultipartUpload",
+ "s3:ListMultipartUploadParts"
+ ],
+ "Resource": [
+ "arn:aws:s3:::{{ kops.state_bucket }}/*"
+ ]
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "s3:ListBucket"
+ ],
+ "Resource": [
+ "arn:aws:s3:::{{ kops.state_bucket }}"
+ ]
+ }
+ ]
+}

roles/velero/files/flux-kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+kind: Kustomization
+metadata:
+ name: velero
+ namespace: flux-system
+spec:
+ interval: 5m0s
+ path: ./platform/velero
+ sourceRef:
+ kind: GitRepository
+ name: tavros
+ validation: client
+ prune: true
+ timeout: 5m0s

roles/velero/files/helm-repo.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+ name: vmware-tanzu
+ namespace: flux-system
+spec:
+ interval: 30m
+ url: https://vmware-tanzu.github.io/helm-charts

roles/velero/files/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+ - helm-repo.yaml
+ - ns.yaml
+ - volume-snapshot-class.yaml
+ - secret-service-account-creds.yaml
+ - release.yaml

roles/velero/files/ns.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: velero

roles/velero/tasks/main.yaml

@@ -0,0 +1,59 @@
+---
+- name: Create Directories
+ file:
+ path: /tmp/{{ cluster_fqdn }}/platform/velero/
+ state: directory
+
+- name: Template Files
+ template:
+ src: "{{ item.name }}.j2"
+ dest: /tmp/{{ cluster_fqdn }}/platform/velero/{{ item.name }}{{ item.type | default('.yaml')}}
+ when: item.condition | default(true)
+ loop:
+ - name: release
+ - name: volume-snapshot-class
+ - name: secret-service-account-creds
+
+- name: Seal Service Acount Creds
+ tags: [ requires_cluster ]
+ when: ('dry-run' not in ansible_run_tags)
+ shell: |
+ kubeseal --format=yaml </tmp/{{ cluster_fqdn }}/platform/velero/secret-service-account-creds.yaml >/tmp/{{ cluster_fqdn }}/platform/velero/secret-service-account-creds.tmp
+ mv /tmp/{{ cluster_fqdn }}/platform/velero/secret-service-account-creds.tmp /tmp/{{ cluster_fqdn }}/platform/velero/secret-service-account-creds.yaml
+
+- name: Copy Files
+ copy:
+ src: "{{ item.name }}"
+ dest: /tmp/{{ cluster_fqdn }}/platform/velero/{{ item.name }}
+ when: item.condition | default(true)
+ loop:
+ - name: helm-repo.yaml
+ - name: ns.yaml
+ - name: kustomization.yaml
+
+- name: Create Backup Blob Container
+ when: ('dry-run' not in ansible_run_tags) and kubernetes_cluster.cloud_provider == 'aks'
+ collections:
+ - azure.azcollection
+ azure_rm_storageblob:
+ resource_group: "{{ aks.resource_group }}"
+ storage_account_name: "{{ aks.storage_account_name }}"
+ container: "backups"
+
+- name: Apply Resources
+ tags: [ requires_cluster ]
+ when: ('dry-run' not in ansible_run_tags)
+ loop: "{{ lookup('ms3_inc.tavros.kustomize', '/tmp/' + cluster_fqdn + '/platform/velero/', reorder='none') }}"
+ loop_control:
+ label: "{{ item.kind }}/{{ item.metadata.name | default('unnamed')}}"
+ ms3_inc.tavros.kube:
+ kubeconfig: '~/.kube/config'
+ definition: "{{ item }}"
+ wait: true
+ wait_condition: "{{ wait_conditions[item.kind] | default(omit) }}"
+ wait_timeout: 900
+
+- name: Template flux-kustomization
+ copy:
+ src: flux-kustomization.yaml
+ dest: /tmp/{{ cluster_fqdn }}/platform/flux-system/watches/velero.yaml

roles/velero/templates/aws-sp-creds.j2

@@ -0,0 +1,2 @@
+aws_access_key_id={{ kops.sp.client_id }}
+aws_secret_access_key={{ kops.sp.client_secret }}

roles/velero/templates/azure-sp-creds.j2

@@ -0,0 +1,6 @@
+AZURE_SUBSCRIPTION_ID={{ aks.subscription_id }}
+AZURE_TENANT_ID={{ aks.tenant_id }}
+AZURE_CLIENT_ID={{ aks.sp.client_id }}
+AZURE_CLIENT_SECRET={{ aks.sp.client_secret }}
+AZURE_RESOURCE_GROUP={{ aks.resource_group }}
+AZURE_CLOUD_NAME=AzurePublicCloud