Cloudformation stacks End to End tests - test suite structure

.github/workflows/pull_request.yaml

@@ -3,6 +3,7 @@ name: Pull Request Workflow
 on:
   pull_request:
     branches:
+      - develop
       - main
 
 jobs:

.github/workflows/run-e2e-tests.yaml

@@ -0,0 +1,60 @@
+name: E2E Test Workflow
+
+on:
+  pull_request:
+    branches:
+      - develop
+      - main
+
+jobs:
+  run-e2e-tests:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+
+      - name: Install AWS SAM CLI
+        run: |
+          pip install aws-sam-cli
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ secrets.AWS_E2E_ROLE }}
+          aws-region: us-east-1
+
+      - name: Run e2e tests
+        env:
+          NEW_RELIC_USER_KEY: ${{ secrets.NEW_RELIC_USER_KEY }}
+          NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+          S3_BUCKET: unified-lambda-e2e-test-templates
+        run: |
+          cd e2e-tests/
+          ./build-templates.sh
+
+          echo "Running s3 and cloudwatch trigger tests parallely"
+          ./lambda-cloudwatch-trigger.sh &
+          pid1=$!
+          echo "Testing setting up cloudwatch trigger with PID: $pid1"
+
+          ./lambda-s3-trigger.sh &
+          pid2=$!
+          echo "Testing setting up s3 trigger with PID: $pid2"
+
+          wait $pid1
+          wait $pid2
+
+      - name: Delete Resources
+        env:
+          S3_BUCKET: unified-lambda-e2e-test-templates
+        run:
+          aws s3 rm "s3://$S3_BUCKET" --recursive

e2e-tests/build-templates.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# add all templates for subsequent test cases
+# make packaging and publishing of templates more efficient so every time new al2 env is not spun up
+TEMPLATES=("lambda-template.yaml")
+
+source config-file.cfg
+
+for TEMPLATE_FILE in "${TEMPLATES[@]}"; do
+
+  BASE_NAME=$(basename "$TEMPLATE_FILE" .yaml)
+  BUILD_DIR="$BUILD_DIR_BASE/$BASE_NAME"
+
+  sam build -u --template-file "../$TEMPLATE_FILE" --build-dir "$BUILD_DIR"
+  sam package --s3-bucket "$S3_BUCKET" --template-file "$BUILD_DIR/template.yaml" --output-template-file "$BUILD_DIR/$TEMPLATE_FILE"
+
+done

e2e-tests/common-scripts.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+source config-file.cfg
+
+validate_stack_deployment_status() {
+  stack_name=$1
+
+  echo "Validating stack deployment status for stack name: $stack_name"
+
+  stack_status=$(aws cloudformation describe-stacks --stack-name "$stack_name" --query "Stacks[0].StackStatus" --output text)
+  if [[ "$stack_status" == "ROLLBACK_COMPLETE" || "$stack_status" == "ROLLBACK_FAILED" || "$stack_status" == "CREATE_FAILED"  || "$stack_status" == "UPDATE_FAILED" ]]; then
+    echo "Stack $stack_name failed to be created and rolled back."
+    failure_reason=$(aws cloudformation describe-stack-events --stack-name "$stack_name" --query "StackEvents[?ResourceStatus==\`$stack_status\`].ResourceStatusReason" --output text)
+    exit_with_error "Stack $stack_name failed to be created. Failure reason: $failure_reason"
+  else
+    echo "Stack $stack_name was created successfully."
+  fi
+}
+
+delete_stack() {
+  stack_name=$1
+
+  aws cloudformation delete-stack --stack-name "$stack_name"
+
+  echo "Initiated deletion of stack: $stack_name"
+
+  stack_status=$(aws cloudformation describe-stacks --stack-name "$stack_name" --query 'Stacks[0].StackStatus' --output text)
+
+  # delete stack with exponential back off retires
+  max_sleep_time=300  # Cap sleep time to 5 minutes
+  sleep_time=30
+
+  while [[ $stack_status == "DELETE_IN_PROGRESS" ]]; do
+    echo "Stack $stack_name is still being deleted..."
+
+    sleep $sleep_time
+    if (( sleep_time < max_sleep_time )); then
+      sleep_time=$(( sleep_time * 2 ))
+    fi
+
+    stack_status=$(aws cloudformation describe-stacks --stack-name "$stack_name" --query 'Stacks[0].StackStatus' --output text 2>/dev/null || true)
+  done
+
+  if [ -z "$stack_status" ]; then
+    echo "Stack $stack_name has been successfully deleted."
+  elif [ "$stack_status" == "DELETE_FAILED" ]; then
+    echo "Failed to delete stack $stack_name."
+  else
+    echo "Unexpected stack status: $stack_status."
+  fi
+}
+
+exit_with_error() {
+  echo "Error: $1"
+  exit 1
+}
+
+get_lambda_function_arn() {
+  stack_name=$1
+
+  lambda_physical_id=$(aws cloudformation describe-stack-resources \
+                    --stack-name "$stack_name" \
+                    --logical-resource-id "$LAMBDA_LOGICAL_RESOURCE_ID" \
+                    --query "StackResources[0].PhysicalResourceId" \
+                    --output text
+  )
+
+  lambda_function_arn=$(aws lambda get-function --function-name "$lambda_physical_id" \
+                  --query "Configuration.FunctionArn" \
+                  --output text
+  )
+
+  echo "$lambda_function_arn"
+}

e2e-tests/config-file.cfg

@@ -0,0 +1,18 @@
+# test resources
+NEW_RELIC_REGION=US
+NEW_RELIC_ACCOUNT_ID=2813435
+
+# build constants
+BUILD_DIR_BASE=.aws-sam/build
+
+# template constants
+LAMBDA_TEMPLATE=lambda-template.yaml
+LAMBDA_TEMPLATE_BUILD_DIR=.aws-sam/build/lambda-template
+LAMBDA_LOGICAL_RESOURCE_ID=NewRelicLogsServerlessLogForwarder
+
+# deployment constants
+S3_BUCKET_NAME=aws-unified-lambda-e2e-test-bucket
+S3_BUCKET_PREFIX=LogFolder
+LOG_GROUP_NAME=aws-unified-lambda-e2e-test-log-group
+LOG_GROUP_FILTER_PATTERN=ERROR
+

e2e-tests/lambda-cloudwatch-trigger.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+source common-scripts.sh
+source config-file.cfg
+
+# test case constants
+CLOUDWATCH_TRIGGER_CASE=e2e-cloudwatch-trigger-stack
+
+deploy_cloudwatch_trigger_stack() {
+  template_file=$1
+  stack_name=$2
+  license_key=$3
+  new_relic_region=$4
+  new_relic_account_id=$5
+  secret_license_key=$6
+  log_group_config=$7
+  common_attributes=$8
+
+  echo "Deploying cloudwatch trigger stack with name: $stack_name"
+
+  sam deploy \
+    --template-file "$template_file" \
+    --stack-name "$stack_name" \
+    --parameter-overrides \
+      LicenseKey="$license_key" \
+      NewRelicRegion="$new_relic_region" \
+      NewRelicAccountId="$new_relic_account_id" \
+      StoreNRLicenseKeyInSecretManager="$secret_license_key" \
+      S3BucketNames="''" \
+      LogGroupConfig="$log_group_config" \
+      CommonAttributes="$common_attributes" \
+    --capabilities CAPABILITY_IAM
+}
+
+validate_lambda_subscription_created() {
+  # this function fetches cloudwatch subscriptions and
+  # validates if lambda subscription filter is configured
+  stack_name=$1
+  log_group_name=$2
+  log_group_filter=$3
+
+  echo "Validating cloudwatch lambda subscription for stack name: $stack_name, log group name: $log_group_name, and log group filter: $log_group_filter"
+
+  lambda_function_arn=$(get_lambda_function_arn "$stack_name")
+
+  subscriptions=$(aws logs describe-subscription-filters --log-group-name "$log_group_name" --query 'subscriptionFilters[*].[destinationArn, filterPattern]' --output text)
+
+  if echo "$subscriptions" | grep -q "$lambda_function_arn" && echo "$subscriptions" | grep -q "$log_group_filter"; then
+    echo "Lambda function $lambda_function_arn is subscribed to log group: $log_group_name with filter: $log_group_filter"
+  else
+    exit_with_error "Lambda function $lambda_function_arn is not subscribed to log group: $log_group_name"
+  fi
+
+}
+
+cat <<EOF > cloudwatch-parameter.json
+'[{"LogGroupName":"$LOG_GROUP_NAME","FilterPattern":"$LOG_GROUP_FILTER_PATTERN"}]'
+EOF
+LOG_GROUP_NAMES=$(<cloudwatch-parameter.json)
+echo "Testing for log group configuration JSON: $(<cloudwatch-parameter.json)"
+
+deploy_cloudwatch_trigger_stack "$LAMBDA_TEMPLATE_BUILD_DIR/$LAMBDA_TEMPLATE" "$CLOUDWATCH_TRIGGER_CASE" "$NEW_RELIC_LICENSE_KEY" "$NEW_RELIC_REGION" "$NEW_RELIC_ACCOUNT_ID" "false"  "$LOG_GROUP_NAMES" "''"
+validate_stack_deployment_status "$CLOUDWATCH_TRIGGER_CASE"
+validate_lambda_subscription_created "$CLOUDWATCH_TRIGGER_CASE" "$LOG_GROUP_NAME" "$LOG_GROUP_FILTER_PATTERN"
+delete_stack "$CLOUDWATCH_TRIGGER_CASE"

e2e-tests/lambda-s3-trigger.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+source common-scripts.sh
+source config-file.cfg
+
+# test case constants
+S3_TRIGGER_CASE=e2e-s3-trigger-stack
+
+deploy_s3_trigger_stack() {
+  template_file=$1
+  stack_name=$2
+  license_key=$3
+  new_relic_region=$4
+  new_relic_account_id=$5
+  secret_license_key=$6
+  s3_bucket_names=$7
+  common_attributes=$8
+
+  echo "Deploying s3 trigger stack with name: $stack_name"
+
+  sam deploy \
+    --template-file "$template_file" \
+    --stack-name "$stack_name" \
+    --parameter-overrides \
+      LicenseKey="$license_key" \
+      NewRelicRegion="$new_relic_region" \
+      NewRelicAccountId="$new_relic_account_id" \
+      StoreNRLicenseKeyInSecretManager="$secret_license_key" \
+      S3BucketNames="$s3_bucket_names" \
+      LogGroupConfig="''" \
+      CommonAttributes="$common_attributes" \
+    --capabilities CAPABILITY_IAM
+}
+
+validate_lambda_s3_trigger_created() {
+  # this function fetches bucket configurations and
+  # validates if lambda event notification is configured
+  stack_name=$1
+  bucket_name=$2
+  bucket_prefix=$3
+
+  echo "Validating s3 lambda trigger event for stack name: $stack_name, bucket name: $bucket_name, and prefix: $bucket_prefix"
+
+  lambda_function_arn=$(get_lambda_function_arn "$stack_name")
+
+  notification_configuration=$(aws s3api get-bucket-notification-configuration --bucket "$bucket_name")
+
+  lambda_configurations=$(echo "$notification_configuration" |
+      jq --arg lambda "$lambda_function_arn" --arg prefix "$bucket_prefix" '
+      .LambdaFunctionConfigurations[] |
+      select(.LambdaFunctionArn == $lambda and (.Filter.Key.FilterRules[]? | select(.Name == "Prefix" and .Value == $prefix)?))')
+
+  if [ -n "$lambda_configurations" ]; then
+      echo "S3 triggers with prefix '$bucket_prefix' found for Lambda function $lambda_function_arn on bucket $bucket_name:"
+      echo "$lambda_configurations" | jq '.'
+  else
+      exit_with_error "No S3 triggers with prefix '$bucket_prefix' found for Lambda function $lambda_function_arn on bucket $bucket_name."
+  fi
+}
+
+cat <<EOF > s3-parameter.json
+'[{"bucket":"$S3_BUCKET_NAME","prefix":"$S3_BUCKET_PREFIX"}]'
+EOF
+S3_BUCKET_NAMES=$(<s3-parameter.json)
+echo "Testing for s3 bucket configuration JSON: $(<s3-parameter.json)"
+
+deploy_s3_trigger_stack "$LAMBDA_TEMPLATE_BUILD_DIR/$LAMBDA_TEMPLATE" "$S3_TRIGGER_CASE" "$NEW_RELIC_LICENSE_KEY" "$NEW_RELIC_REGION" "$NEW_RELIC_ACCOUNT_ID" "false" "$S3_BUCKET_NAMES" "''"
+validate_stack_deployment_status "$S3_TRIGGER_CASE"
+validate_lambda_s3_trigger_created "$S3_TRIGGER_CASE" "$S3_BUCKET_NAME" "$S3_BUCKET_PREFIX"
+delete_stack "$S3_TRIGGER_CASE"