diff --git a/.github/workflows/ci-base.yaml b/.github/workflows/ci-base.yaml index ad1685f6..bee710f9 100644 --- a/.github/workflows/ci-base.yaml +++ b/.github/workflows/ci-base.yaml @@ -107,7 +107,7 @@ jobs: args: ${{ env.goreleaser_args }} workdir: distributions/${{ inputs.distribution }} - - name: Extract image version and arch + - name: Extract relevant metadata run: | VERSION=$(echo '${{ steps.goreleaser.outputs.metadata }}' | jq -r '.version') ARCH=$(echo '${{ runner.arch }}' | sed 's/X/amd/g') diff --git a/.github/workflows/ci-nightly.yaml b/.github/workflows/ci-nightly.yaml index 702b298e..a7bad7d5 100644 --- a/.github/workflows/ci-nightly.yaml +++ b/.github/workflows/ci-nightly.yaml @@ -121,7 +121,49 @@ jobs: version: '~> v2' args: --skip=announce,validate --clean --timeout 2h --config .goreleaser-nightly.yaml workdir: distributions/${{ matrix.distribution }} + - name: Extract Docker Manifest SHA + id: extract_docker_manifest_sha + run: | + echo "docker_manifest_sha=$(echo '${{ steps.goreleaser.outputs.artifacts }}' | + yq -r '.[] | select(.type == "Docker Manifest") | .extra.Digest')" >> $GITHUB_OUTPUT + outputs: + docker_manifest_sha: ${{ steps.extract_docker_manifest_sha.outputs.docker_manifest_sha }} + + deploy-nightly: + name: Deploy Nightly + uses: ./.github/workflows/terraform.yaml + if: ${{ !github.event.act }} + needs: publish + strategy: + matrix: + distribution: + - nrdot-collector-host + - nrdot-collector-k8s + with: + branch: ${{ github.ref }} + tf_work_subdir: nightly + workspace: 'nightly-${{ matrix.distribution }}' + execute_plan: true + execute_apply: true + distro: ${{ matrix.distribution }} + nightly_docker_manifest_sha: ${{ needs.publish.outputs.docker_manifest_sha }} + secrets: + aws_access_key_id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCESS_KEY_ID }} + aws_secret_access_key: ${{secrets.OTELCOMM_AWS_TEST_ACC_SECRET_ACCESS_KEY}} + aws_account_id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }} + nr_backend_url: ${{secrets.NR_STAGING_BACKEND_URL}} + nr_ingest_key: ${{ secrets.OTELCOMM_NR_INGEST_KEY }} + test-nightly: + name: Test Nightly + runs-on: ubuntu-latest + needs: deploy-nightly + strategy: + matrix: + distribution: + - nrdot-collector-host + - nrdot-collector-k8s + steps: - name: Run nightly tests for ${{ matrix.distribution }} run: | NR_API_KEY=${{ secrets.OTELCOMM_NR_API_KEY }} \ diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b610e12..afd3199b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,5 +43,3 @@ jobs: aws_access_key_id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCESS_KEY_ID }} aws_secret_access_key: ${{secrets.OTELCOMM_AWS_TEST_ACC_SECRET_ACCESS_KEY}} aws_account_id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }} - nr_backend_url: ${{secrets.NR_STAGING_BACKEND_URL}} - nr_ingest_key: ${{ secrets.OTELCOMM_NR_INGEST_KEY }} diff --git a/.github/workflows/terraform-debug.yaml b/.github/workflows/terraform-debug.yaml deleted file mode 100644 index 60d1b7e7..00000000 --- a/.github/workflows/terraform-debug.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: 🪲 Terraform Debug - -on: - workflow_dispatch: - inputs: - branch: - description: "Branch to run the workflow on" - type: string - required: true - tf_work_subdir: - description: "Subdir of ./test/terraform/ to execute tf in. Used to separate different tf use cases" - type: string - required: true - execute_plan: - description: "Whether to run `terraform plan` or not" - type: boolean - required: true - default: false - execute_apply: - description: "Whether to run `terraform apply` or not" - type: boolean - required: true - default: false - execute_plan_destroy: - description: "Whether to run `terraform plan -destroy` or not" - type: boolean - required: true - default: false - execute_destroy: - description: "Whether to run `terraform destroy` or not" - type: boolean - required: true - default: false - tf_log: - description: "Terraform log level (TRACE, DEBUG, INFO, WARN or ERROR)" - type: string - required: false - default: "WARN" - -jobs: - terraform-debug: - uses: ./.github/workflows/terraform.yaml - with: - branch: ${{ inputs.branch }} - tf_work_subdir: ${{inputs.tf_work_subdir}} - execute_plan: ${{ inputs.execute_plan }} - execute_apply: ${{ inputs.execute_apply }} - execute_plan_destroy: ${{ inputs.execute_plan_destroy }} - execute_destroy: ${{ inputs.execute_destroy }} - tf_log: ${{ inputs.tf_log }} - secrets: - aws_access_key_id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCESS_KEY_ID }} - aws_secret_access_key: ${{secrets.OTELCOMM_AWS_TEST_ACC_SECRET_ACCESS_KEY}} - aws_account_id: ${{ secrets.OTELCOMM_AWS_TEST_ACC_ACCOUNT_ID }} - nr_backend_url: ${{secrets.NR_STAGING_BACKEND_URL}} - nr_ingest_key: ${{ secrets.OTELCOMM_NR_INGEST_KEY }} diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 2a1e460c..8e0ea49b 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -11,6 +11,11 @@ on: description: "Subdir of ./test/terraform/ to execute tf in. Used to separate different tf use cases" type: string required: true + workspace: + description: "Terraform workspace to use" + type: string + required: false + default: default execute_plan: description: "Whether to run `terraform plan` or not" type: boolean @@ -36,6 +41,16 @@ on: type: string required: false default: "INFO" + distro: + description: "List of distributions to test" + type: string + required: false + default: "nrdot-collector-host" + nightly_docker_manifest_sha: + description: "SHA256 to identify nightly docker manifest to use" + type: string + required: false + default: "docker_manifest_sha_placeholder" secrets: aws_access_key_id: description: "AWS credentials for tf with permission to assume resource-provider" @@ -48,11 +63,10 @@ on: required: true nr_backend_url: description: "NR backend url for test purposes" - required: true + required: false nr_ingest_key: description: "NR ingest key for test purposes" - required: true - + required: false jobs: terraform: @@ -66,6 +80,8 @@ jobs: TF_VAR_aws_account_id: ${{ secrets.aws_account_id }} TF_VAR_nr_backend_url: ${{ secrets.nr_backend_url }} TF_VAR_nr_ingest_key: ${{ secrets.nr_ingest_key }} + TF_VAR_distros: ${{ inputs.distros}} + TF_VAR_nightly_docker_manifest_sha: ${{ inputs.nightly_docker_manifest_sha }} steps: - name: Checkout repository @@ -82,6 +98,13 @@ jobs: working-directory: ./test/terraform/${{ inputs.tf_work_subdir }} run: terraform init -backend-config='role_arn=arn:aws:iam::${{ secrets.aws_account_id }}:role/resource-provisioner' -backend-config='bucket=nr-terraform-states-${{ secrets.aws_account_id}}' + - name: Select Workspace + working-directory: ./test/terraform/${{ inputs.tf_work_subdir }} + env: + TF_CLI_ARGS: "" # workspace does not properly support global options + run: | + terraform workspace select -or-create=true ${{ inputs.workspace }} + - name: Terraform Plan if: ${{ inputs.execute_plan }} working-directory: ./test/terraform/${{ inputs.tf_work_subdir }} diff --git a/test/terraform/nightly/main.tf b/test/terraform/nightly/main.tf new file mode 100644 index 00000000..14c01424 --- /dev/null +++ b/test/terraform/nightly/main.tf @@ -0,0 +1,65 @@ +locals { + test_spec = yamldecode(file("${path.module}/../../../distributions/${var.distro}/test-spec.yaml")) + releases_bucket_name = "nr-releases" + required_permissions_boundary_arn_for_new_roles = "arn:aws:iam::${var.aws_account_id}:policy/resource-provisioner-boundary" +} + +resource "random_string" "deploy_id" { + length = 6 + special = false +} + + +data "aws_ecr_repository" "ecr_repo" { + name = var.distro +} + +resource "helm_release" "ci_e2e_nightly" { + name = "ci-e2etest-nightly-${var.distro}" + chart = "../../charts/nr_backend" + + create_namespace = true + namespace = "nightly-${var.distro}" + + set { + name = "image.repository" + value = data.aws_ecr_repository.ecr_repo[var.distro].repository_url + } + + set { + name = "image.tag" + value = "nightly@${var.nightly_docker_manifest_sha}" + } + + set { + name = "image.pullPolicy" + value = "Always" + } + + set { + name = "secrets.nrBackendUrl" + value = var.nr_backend_url + } + + set { + name = "secrets.nrIngestKey" + value = var.nr_ingest_key + } + + set { + name = "collector.hostname" + value = "${var.test_environment}-${random_string.deploy_id.result}-${var.distro}-k8s_node" + } +} + +module "ci_e2e_ec2" { + count = local.test_spec.nightly.ec2.enabled ? 1 : 0 + source = "../modules/ec2" + releases_bucket_name = local.releases_bucket_name + collector_distro = var.distro + nr_ingest_key = var.nr_ingest_key + # reuse vpc to avoid having to pay for second NAT gateway for this simple use case + vpc_id = data.aws_eks_cluster.eks_cluster.vpc_config[0].vpc_id + deploy_id = random_string.deploy_id.result + permission_boundary = local.required_permissions_boundary_arn_for_new_roles +} diff --git a/test/terraform/nightly/providers.tf b/test/terraform/nightly/providers.tf new file mode 100644 index 00000000..564fdb16 --- /dev/null +++ b/test/terraform/nightly/providers.tf @@ -0,0 +1,47 @@ +terraform { + required_version = "1.9.8" + required_providers { + aws = { + version = "5.81.0" + } + helm = { + version = "2.17.0" + } + } +} + +terraform { + backend "s3" { + encrypt = true + dynamodb_table = "terraform-states-lock" + region = "us-east-1" + key = "newrelic/opentelemetry-collector-releases/permanent/terraform.tfstate" + # 'bucket' and 'role_arn' provided via '-backend-config' + } +} + +provider "aws" { + region = var.aws_region + allowed_account_ids = [var.aws_account_id] + # expect AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY as env vars + + assume_role { + role_arn = "arn:aws:iam::${var.aws_account_id}:role/resource-provisioner" + } +} + +data "aws_eks_cluster" "eks_cluster" { + name = "aws-ci-e2etest" +} + +data "aws_eks_cluster_auth" "eks_cluster_auth" { + name = "aws-ci-e2etest" +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.eks_cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.eks_cluster.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.eks_cluster_auth.token + } +} diff --git a/test/terraform/nightly/vars.tf b/test/terraform/nightly/vars.tf new file mode 100644 index 00000000..3290999c --- /dev/null +++ b/test/terraform/nightly/vars.tf @@ -0,0 +1,39 @@ +variable "aws_account_id" { + type = string + description = "AWS account id to deploy to" +} + +variable "aws_region" { + type = string + description = "AWS region to deploy to" + default = "us-east-1" +} + +variable "distro" { + description = "Distro to test during nightly" + type = string +} + +variable "nightly_docker_manifest_sha" { + description = "SHA of the nightly docker manifest" + type = string +} + +variable "nr_backend_url" { + type = string + description = "NR endpoint used in test cluster" + sensitive = true +} + +variable "nr_ingest_key" { + type = string + description = "NR ingest key used in test cluster" + sensitive = true +} + +variable "test_environment" { + type = string + description = "Name of test environment to distinguish entities" + default = "nightly" +} + diff --git a/test/terraform/permanent/main.tf b/test/terraform/permanent/main.tf index 3651df1c..27ecead8 100644 --- a/test/terraform/permanent/main.tf +++ b/test/terraform/permanent/main.tf @@ -3,10 +3,6 @@ locals { for _, v in fileset(path.module, "../../../distributions/*/**") : regex("../../../distributions/([^/]*).*", dirname(v)) ]))) - test_specs = { - for distro in local.distros : - distro => yamldecode(file("${path.module}/../../../distributions/${distro}/test-spec.yaml")) - } releases_bucket_name = "nr-releases" required_permissions_boundary_arn_for_new_roles = "arn:aws:iam::${var.aws_account_id}:policy/resource-provisioner-boundary" } @@ -78,62 +74,4 @@ module "s3_bucket" { data "aws_eks_cluster_auth" "this" { name = module.ci_e2e_cluster.cluster_name -} - -resource "random_string" "deploy_id" { - length = 6 - special = false -} - -resource "helm_release" "ci_e2e_nightly" { - for_each = local.distros - depends_on = [module.ci_e2e_cluster, module.ecr] - - name = "ci-e2etest-nightly" - chart = "../../charts/nr_backend" - - create_namespace = true - namespace = "nightly-${each.key}" - - set { - name = "image.repository" - value = module.ecr[each.key].repository_url - } - - set { - name = "image.tag" - value = "nightly" - } - - set { - name = "image.pullPolicy" - value = "Always" - } - - set { - name = "secrets.nrBackendUrl" - value = var.nr_backend_url - } - - set { - name = "secrets.nrIngestKey" - value = var.nr_ingest_key - } - - set { - name = "collector.hostname" - value = "${var.test_environment}-${random_string.deploy_id.result}-${each.key}-k8s_node" - } -} - -module "ci_e2e_ec2" { - for_each = toset([for distro in local.distros : distro if local.test_specs[distro].nightly.ec2.enabled]) - source = "../modules/ec2" - releases_bucket_name = local.releases_bucket_name - collector_distro = each.key - nr_ingest_key = var.nr_ingest_key - # reuse vpc to avoid having to pay for second NAT gateway for this simple use case - vpc_id = module.ci_e2e_cluster.eks_vpc_id - deploy_id = random_string.deploy_id.result - permission_boundary = local.required_permissions_boundary_arn_for_new_roles -} +} \ No newline at end of file diff --git a/test/terraform/permanent/vars.tf b/test/terraform/permanent/vars.tf index bb07305e..dac0cd97 100644 --- a/test/terraform/permanent/vars.tf +++ b/test/terraform/permanent/vars.tf @@ -9,21 +9,5 @@ variable "aws_region" { default = "us-east-1" } -variable "nr_backend_url" { - type = string - description = "NR endpoint used in test cluster" - sensitive = true -} -variable "nr_ingest_key" { - type = string - description = "NR ingest key used in test cluster" - sensitive = true -} - -variable "test_environment" { - type = string - description = "Name of test environment to distinguish entities" - default = "nightly" -}