feat: Terraform CI (#99)

feat: add workflow to: 
- create a Kubernetes cluster
- deploy TCE nodes
- export telemetry data
- clean up the resources created

---------

Signed-off-by: Thiago Nobayashi <foo0x29a@users.noreply.github.com>

.github/workflows/docker_build_push.yml

@@ -1,6 +1,9 @@
-name: Docker_build_push # This is used in the .github/workflows/k8s_perf_tests.yml workflow
+name: Docker_build_push
 
 on:
+  schedule:
+    # Triggers the workflow every day at 5:30 UTC:
+    - cron:  '30 5 * * *'
   push:
     branches: [main]
   pull_request:
@@ -66,3 +69,10 @@ jobs:
         with:
           name: docker_tags
           path: docker/
+
+  k8s:
+    # run only when the workflow was scheduled or a PR is being merged into main
+    if: ${{ (github.event_name != 'pull_request') }}
+    needs: docker
+    uses: toposware/topos/.github/workflows/k8s_e2e_tests.yml@main
+    secrets: inherit

.github/workflows/k8s_e2e_tests.yml

@@ -1,69 +1,114 @@
 name: E2E tests on Kubernetes
 
 on:
-  workflow_run:
-    workflows: [Docker_build_push] # This is the name of the .github/workflows/docker_build_push.yml workflow
-    types:
-      - completed
-
-env:
-  CERT_SPAMMER_IMAGE: ghcr.io/toposware/cert-spammer:main
-  TELEMETRY_ENDPOINT: jaeger.tce.devnet-1.toposware.com:6831
-  SPAM_DURATION_SECONDS: 10
-  TERRAFORM_INPUTS: >
-    -var="cluster_name=${{ github.sha }}"
-    -var="cert_spammer_image=${{ env.CERT_SPAMMER_IMAGE }}"
-    -var="jaeger_endpoint=${{ env.TELEMETRY_ENDPOINT }}"
-    -var="jaeger_service_name=${{ github.sha }}
-    -var="docker_registry_auth_user=${{ github.actor }}"
-    -var="docker_registry_auth_pwd=${{ secrets.GITHUB_TOKEN }}"
+  workflow_call:
 
 jobs:
-  retrieve-docker-tags:
-    name: Retrieve docker tags from triggering workflow
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-    outputs:
-      docker_tags: ${{ steps.docker_tags.outputs.result }}
-    steps:
-      - name: Retrieve docker tags
-        id: docker_tags
-        uses: actions/github-script@v6
-        with:
-          script: |
-            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               run_id: context.payload.workflow_run.id,
-            });
-            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
-              return artifact.name == "docker_tags"
-            })[0];
-            let download = await github.rest.actions.downloadArtifact({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               artifact_id: matchArtifact.id,
-            });
-            return download.data;
-          result-encoding: string
   k8s:
-    name: Deploy k8s cluster and run TCE network
-    needs: retrieve-docker-tags
+    name: Deploy k8s cluster and run Observability Stack
     runs-on: ubuntu-latest
     container:
-      image: hashicorp/terraform:1.2.4
+      image: ghcr.io/toposware/k8s_e2e:latest
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.ROBOT_TOPOSWARE_GH_PACKAGE_TOKEN }}
     permissions:
       contents: read
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Terraform init
-        run: terraform -chdir=./tools/terraform init
-      - name: Terraform plan
-        run: terraform -chdir=./tools/terraform plan ${{ env.TERRAFORM_INPUTS }} -var="tce_node_image=${{ needs.retrieve-docker-tags.outputs.docker_tags }}"
-      - name: Spam certificates for ${{ env.SPAM_DURATION_SECONDS }} seconds
-        uses: jakejarvis/wait-action@master
+
+      - name: Retrieve Docker tags
+        id: docker_tags
+        uses: actions/download-artifact@v3
+        with:
+          name: docker_tags
+
+      - name: Set environment
+        run: |
+          docker_tag=$(head -n 1 docker_tags)
+          echo "DOCKER_TAG=${docker_tag#*:}" >> $GITHUB_ENV
+
+          sha=${{ github.sha }}
+          e2e_id="e2e-${{ github.repository_id }}-${sha:0:6}"
+          echo "E2E_ID=${e2e_id}" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Enable remote modules download
+        run: git config --global url."https://${{ secrets.ROBOT_TOPOSWARE_PRIV_REPOS_TOKEN }}@github.com/".insteadOf "https://github.com/"
+
+      - name: Configure AWS credentials for cicd-devnet-1 account
+        uses: aws-actions/configure-aws-credentials@v1
         with:
-          time: "${{ env.SPAM_DURATION_SECONDS }}s"
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          role-to-assume: arn:aws:iam::367397670706:role/TerraformRole
+          role-session-name: Terraform-TerraformRole-session
+          aws-region: us-east-1
+          role-skip-session-tagging: true
+          role-duration-seconds: 3600
+
+      - name: Terraform init
+        working-directory: tools/terraform/cicd-devnet-1
+        run: |
+          terraform init \
+            -backend-config="key=${{ env.E2E_ID }}.tfstate" \
+            -backend-config=backend.conf
+
+      - name: Terraform apply
+        working-directory: tools/terraform/cicd-devnet-1
+        run: |
+          terraform apply -auto-approve \
+            -var "name=${{ env.E2E_ID }}" \
+            -var-file=terraform.tfvars.default
+
+      - name: Grab results from Terraform
+        working-directory: tools/terraform/cicd-devnet-1
+        run: |
+          filesystem_id=$(terraform output --raw eks_efs_id)
+          echo "FILESYSTEM_ID=${filesystem_id}" >> $GITHUB_ENV
+
+          eks_efs_csi_irsa_arn=$(terraform output --raw eks_efs_csi_irsa_arn)
+          echo "EKS_EFS_CSI_IRSA_ARN=${eks_efs_csi_irsa_arn}" >> $GITHUB_ENV
+
+      - name: Configure kubeconfig
+        working-directory: tools/terraform/cicd-devnet-1
+        run: aws eks update-kubeconfig --region us-east-1 --name $(terraform output --raw cluster_name)
+
+      - name: Helm Install
+        working-directory: tools/
+        env:
+          GITHUB_USERNAME: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.ROBOT_TOPOSWARE_PRIV_REPOS_TOKEN }}
+          DOCKER_TAGS: ${{ env.DOCKER_TAG }}
+          DOCKER_USERNAME: ${{ github.actor }}
+          DOCKER_PASSWORD: ${{ secrets.ROBOT_TOPOSWARE_GH_PACKAGE_TOKEN }}
+          FILESYSTEM_ID: ${{ env.FILESYSTEM_ID }}
+        run: |
+          helmfile destroy && helmfile sync
+
+      - name: Sleep
+        run: sleep 300s
+        shell: bash
+
+      - name: Helm Clean up
+        if: always()
+        working-directory: tools/
+        env:
+          GITHUB_USERNAME: ${{ github.actor }}
+          GITHUB_TOKEN: ${{ secrets.ROBOT_TOPOSWARE_PRIV_REPOS_TOKEN }}
+          DOCKER_TAGS: ${{ env.DOCKER_TAG }}
+          DOCKER_USERNAME: ${{ github.actor }}
+          DOCKER_PASSWORD: ${{ secrets.ROBOT_TOPOSWARE_GH_PACKAGE_TOKEN }}
+          FILESYSTEM_ID: ${{ env.FILESYSTEM_ID }}
+          EKS_EFS_CSI_IRSA_ARN: ${{ env.EKS_EFS_CSI_IRSA_ARN }}
+        run: |
+          helmfile destroy
+
       - name: Terraform destroy
-        run: terraform -chdir=./tools/terraform destroy ${{ env.TERRAFORM_INPUTS }} -var="tce_node_image=${{ needs.retrieve-docker-tags.outputs.docker_tags }}"
+        if: always()
+        working-directory: tools/terraform/cicd-devnet-1
+        run: |
+          terraform destroy -auto-approve \
+            -var "name=${{ env.E2E_ID }}" \
+            -var-file=terraform.tfvars.default

.gitignore

@@ -31,8 +31,8 @@ crash.log
 crash.*.log
 
 # Exclude all .tfvars files, which are likely to contain sensitive data, such as
-# password, private keys, and other secrets. These should not be part of version 
-# control as they are data points which are potentially sensitive and subject 
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
 # to change depending on the environment.
 *.tfvars
 *.tfvars.json

Cargo.toml

@@ -28,7 +28,7 @@ web3 = { version = "0.18", features = ["signing"]}
 
 # Log, Tracing & telemetry
 opentelemetry = { version = "0.18", features = ["rt-tokio", "metrics"] }
-opentelemetry-otlp = { version = " 0.11", features = ["tonic", "metrics"] }
+opentelemetry-otlp = { version = " 0.11", features = ["tonic", "metrics", "tls", "tls-roots"] }
 opentelemetry-jaeger = { version = " 0.17" }
 pretty_env_logger = { version = "0.4", default-features = false }
 tracing = { version = "0.1", default-features = false }

Dockerfile

@@ -52,6 +52,7 @@ FROM debian:bullseye-slim
 ENV TCE_PORT=9090
 ENV USER=topos
 ENV UID=10001
+ENV PATH="${PATH}:/usr/src/app"
 
 RUN adduser \
     --disabled-password \
@@ -67,7 +68,11 @@ WORKDIR /usr/src/app
 COPY --from=build /usr/src/app/target/release/topos .
 COPY tools/init.sh ./init.sh
 
-RUN apt-get update && apt-get install jq -y
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    jq \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
 
 USER topos:topos
 

tools/helmfile.yaml

@@ -0,0 +1,51 @@
+helmDefaults:
+  wait: true
+  timeout: 600
+
+repositories:
+ - name: toposware
+   url: https://raw.githubusercontent.com/toposware/helm-charts/gh-pages/
+   username: {{ requiredEnv "GITHUB_USERNAME"}}
+   password: {{ requiredEnv "GITHUB_TOKEN"}}
+ - name: aws-efs-csi-driver
+   url: https://kubernetes-sigs.github.io/aws-efs-csi-driver
+
+releases:
+- name: aws-efs-csi-driver
+  namespace: kube-system
+  chart: aws-efs-csi-driver/aws-efs-csi-driver
+  version: 2.3.6
+  createNamespace: true
+  set:
+  - name: controller.serviceAccount.annotations.eks\.amazonaws\.com/role-arn
+    value: {{ requiredEnv "EKS_EFS_CSI_IRSA_ARN" }}
+
+- name: tce-tmp-storage
+  namespace: default
+  chart: toposware/tce-tmp-storage
+  version: 0.1.0
+  createNamespace: true
+  needs:
+  - kube-system/aws-efs-csi-driver
+  set:
+  - name: fileSystemId
+    value: {{ requiredEnv "FILESYSTEM_ID" }}
+
+- name: tce-all-in-one
+  namespace: default
+  chart: toposware/tce-all-in-one
+  version: 0.1.21
+  createNamespace: true
+  needs:
+  - tce-tmp-storage
+  set:
+  - name: image.repository
+    value: ghcr.io/toposware/topos
+  - name: image.tag
+    value: {{ requiredEnv "DOCKER_TAG"}}
+  - name: credentials.username
+    value: {{ requiredEnv "DOCKER_USERNAME"}}
+  - name: credentials.password
+    value: {{ requiredEnv "DOCKER_PASSWORD"}}
+  - name: otelEndpoint
+    value: https://otel-collector.telemetry.devnet-1.toposware.com:443