Kbauer/apply helm and ec2 during nightly

.github/workflows/ci-base.yaml

@@ -107,7 +107,7 @@ jobs:
           args: ${{ env.goreleaser_args }}
           workdir: distributions/${{ inputs.distribution }}
 
-      - name: Extract image version and arch
+      - name: Extract relevant metadata
         run: |
           VERSION=$(echo '${{ steps.goreleaser.outputs.metadata }}' | jq -r '.version')
           ARCH=$(echo '${{ runner.arch }}' | sed 's/X/amd/g')

.github/workflows/ci-nightly.yaml

@@ -121,7 +121,49 @@ jobs:
           version: '~> v2'
           args: --skip=announce,validate --clean --timeout 2h --config .goreleaser-nightly.yaml
           workdir: distributions/${{ matrix.distribution }}
+      - name: Extract Docker Manifest SHA
+        id: extract_docker_manifest_sha
+        run: |
+          echo "docker_manifest_sha=$(echo '${{ steps.goreleaser.outputs.artifacts }}' |
+            yq -r '.[] | select(.type == "Docker Manifest") | .extra.Digest')" >> $GITHUB_OUTPUT
+    outputs:
+      docker_manifest_sha: ${{ steps.extract_docker_manifest_sha.outputs.docker_manifest_sha }}
+
+  deploy-nightly:
+    name: Deploy Nightly
+    uses: ./.github/workflows/terraform.yaml
+    if: ${{ !github.event.act }}
+    needs: publish
+    strategy:
+      matrix:
+        distribution:
+          - nrdot-collector-host
+          - nrdot-collector-k8s
+    with:
+      branch: ${{ github.ref }}
+      tf_work_subdir: nightly
+      workspace: 'nightly-${{ matrix.distribution }}'
+      execute_plan: true
+      execute_apply: true
+      distro: ${{ matrix.distribution }}
+      nightly_docker_manifest_sha: ${{ needs.publish.outputs.docker_manifest_sha }}
+    secrets:
+      aws_access_key_id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCESS_KEY_ID }}
+      aws_secret_access_key: ${{secrets.OTELCOMM_AWS_TEST_ACC_SECRET_ACCESS_KEY}}
+      aws_account_id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}
+      nr_backend_url: ${{secrets.NR_STAGING_BACKEND_URL}}
+      nr_ingest_key: ${{ secrets.OTELCOMM_NR_INGEST_KEY }}
 
+  test-nightly:
+    name: Test Nightly
+    runs-on: ubuntu-latest
+    needs: deploy-nightly
+    strategy:
+      matrix:
+        distribution:
+          - nrdot-collector-host
+          - nrdot-collector-k8s
+    steps:
       - name: Run nightly tests for ${{ matrix.distribution }}
         run: |
           NR_API_KEY=${{ secrets.OTELCOMM_NR_API_KEY }} \

.github/workflows/ci.yaml

@@ -43,5 +43,3 @@ jobs:
       aws_access_key_id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCESS_KEY_ID }}
       aws_secret_access_key: ${{secrets.OTELCOMM_AWS_TEST_ACC_SECRET_ACCESS_KEY}}
       aws_account_id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }}
-      nr_backend_url: ${{secrets.NR_STAGING_BACKEND_URL}}
-      nr_ingest_key: ${{ secrets.OTELCOMM_NR_INGEST_KEY }}

.github/workflows/terraform.yaml

@@ -11,6 +11,11 @@ on:
         description: "Subdir of ./test/terraform/ to execute tf in. Used to separate different tf use cases"
         type: string
         required: true
+      workspace:
+        description: "Terraform workspace to use"
+        type: string
+        required: false
+        default: default
       execute_plan:
         description: "Whether to run `terraform plan` or not"
         type: boolean
@@ -36,6 +41,16 @@ on:
         type: string
         required: false
         default: "INFO"
+      distro:
+        description: "List of distributions to test"
+        type: string
+        required: false
+        default: "nrdot-collector-host"
+      nightly_docker_manifest_sha:
+        description: "SHA256 to identify nightly docker manifest to use"
+        type: string
+        required: false
+        default: "docker_manifest_sha_placeholder"
     secrets:
       aws_access_key_id:
         description: "AWS credentials for tf with permission to assume resource-provider"
@@ -48,11 +63,10 @@ on:
         required: true
       nr_backend_url:
         description: "NR backend url for test purposes"
-        required: true
+        required: false
       nr_ingest_key:
         description: "NR ingest key for test purposes"
-        required: true
-
+        required: false
 
 jobs:
   terraform:
@@ -66,6 +80,8 @@ jobs:
       TF_VAR_aws_account_id: ${{ secrets.aws_account_id }}
       TF_VAR_nr_backend_url: ${{ secrets.nr_backend_url }}
       TF_VAR_nr_ingest_key: ${{ secrets.nr_ingest_key }}
+      TF_VAR_distros: ${{ inputs.distros}}
+      TF_VAR_nightly_docker_manifest_sha: ${{ inputs.nightly_docker_manifest_sha }}
 
     steps:
       - name: Checkout repository
@@ -82,6 +98,13 @@ jobs:
         working-directory: ./test/terraform/${{ inputs.tf_work_subdir }}
         run: terraform init -backend-config='role_arn=arn:aws:iam::${{ secrets.aws_account_id }}:role/resource-provisioner' -backend-config='bucket=nr-terraform-states-${{ secrets.aws_account_id}}'
 
+      - name: Select Workspace
+        working-directory: ./test/terraform/${{ inputs.tf_work_subdir }}
+        env:
+          TF_CLI_ARGS: "" # workspace does not properly support global options
+        run: |
+          terraform workspace select -or-create=true ${{ inputs.workspace }} 
+
       - name: Terraform Plan
         if: ${{ inputs.execute_plan }}
         working-directory: ./test/terraform/${{ inputs.tf_work_subdir }}

test/terraform/nightly/main.tf

@@ -0,0 +1,65 @@
+locals {
+  test_spec                                       = yamldecode(file("${path.module}/../../../distributions/${var.distro}/test-spec.yaml"))
+  releases_bucket_name                            = "nr-releases"
+  required_permissions_boundary_arn_for_new_roles = "arn:aws:iam::${var.aws_account_id}:policy/resource-provisioner-boundary"
+}
+
+resource "random_string" "deploy_id" {
+  length  = 6
+  special = false
+}
+
+
+data "aws_ecr_repository" "ecr_repo" {
+  name = var.distro
+}
+
+resource "helm_release" "ci_e2e_nightly" {
+  name  = "ci-e2etest-nightly-${var.distro}"
+  chart = "../../charts/nr_backend"
+
+  create_namespace = true
+  namespace        = "nightly-${var.distro}"
+
+  set {
+    name  = "image.repository"
+    value = data.aws_ecr_repository.ecr_repo[var.distro].repository_url
+  }
+
+  set {
+    name  = "image.tag"
+    value = "nightly@${var.nightly_docker_manifest_sha}"
+  }
+
+  set {
+    name  = "image.pullPolicy"
+    value = "Always"
+  }
+
+  set {
+    name  = "secrets.nrBackendUrl"
+    value = var.nr_backend_url
+  }
+
+  set {
+    name  = "secrets.nrIngestKey"
+    value = var.nr_ingest_key
+  }
+
+  set {
+    name  = "collector.hostname"
+    value = "${var.test_environment}-${random_string.deploy_id.result}-${var.distro}-k8s_node"
+  }
+}
+
+module "ci_e2e_ec2" {
+  count                = local.test_spec.nightly.ec2.enabled ? 1 : 0
+  source               = "../modules/ec2"
+  releases_bucket_name = local.releases_bucket_name
+  collector_distro     = var.distro
+  nr_ingest_key        = var.nr_ingest_key
+  # reuse vpc to avoid having to pay for second NAT gateway for this simple use case
+  vpc_id              = data.aws_eks_cluster.eks_cluster.vpc_config[0].vpc_id
+  deploy_id           = random_string.deploy_id.result
+  permission_boundary = local.required_permissions_boundary_arn_for_new_roles
+}

test/terraform/nightly/providers.tf

@@ -0,0 +1,47 @@
+terraform {
+  required_version = "1.9.8"
+  required_providers {
+    aws = {
+      version = "5.81.0"
+    }
+    helm = {
+      version = "2.17.0"
+    }
+  }
+}
+
+terraform {
+  backend "s3" {
+    encrypt        = true
+    dynamodb_table = "terraform-states-lock"
+    region         = "us-east-1"
+    key            = "newrelic/opentelemetry-collector-releases/permanent/terraform.tfstate"
+    # 'bucket' and 'role_arn' provided via '-backend-config'
+  }
+}
+
+provider "aws" {
+  region              = var.aws_region
+  allowed_account_ids = [var.aws_account_id]
+  # expect AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY as env vars
+
+  assume_role {
+    role_arn = "arn:aws:iam::${var.aws_account_id}:role/resource-provisioner"
+  }
+}
+
+data "aws_eks_cluster" "eks_cluster" {
+  name = "aws-ci-e2etest"
+}
+
+data "aws_eks_cluster_auth" "eks_cluster_auth" {
+  name = "aws-ci-e2etest"
+}
+
+provider "helm" {
+  kubernetes {
+    host                   = data.aws_eks_cluster.eks_cluster.endpoint
+    cluster_ca_certificate = base64decode(data.aws_eks_cluster.eks_cluster.certificate_authority[0].data)
+    token                  = data.aws_eks_cluster_auth.eks_cluster_auth.token
+  }
+}

test/terraform/nightly/vars.tf

@@ -0,0 +1,39 @@
+variable "aws_account_id" {
+  type        = string
+  description = "AWS account id to deploy to"
+}
+
+variable "aws_region" {
+  type        = string
+  description = "AWS region to deploy to"
+  default     = "us-east-1"
+}
+
+variable "distro" {
+  description = "Distro to test during nightly"
+  type        = string
+}
+
+variable "nightly_docker_manifest_sha" {
+  description = "SHA of the nightly docker manifest"
+  type        = string
+}
+
+variable "nr_backend_url" {
+  type        = string
+  description = "NR endpoint used in test cluster"
+  sensitive   = true
+}
+
+variable "nr_ingest_key" {
+  type        = string
+  description = "NR ingest key used in test cluster"
+  sensitive   = true
+}
+
+variable "test_environment" {
+  type        = string
+  description = "Name of test environment to distinguish entities"
+  default     = "nightly"
+}
+