diff --git a/.github/workflows/appsignals-e2e-ec2-test.yml b/.github/workflows/appsignals-e2e-ec2-test.yml index 1c6c4ee8d8..4f1acfef68 100644 --- a/.github/workflows/appsignals-e2e-ec2-test.yml +++ b/.github/workflows/appsignals-e2e-ec2-test.yml @@ -21,8 +21,8 @@ env: # It is not redundant AWS_DEFAULT_REGION: ${{ inputs.aws-region }} TEST_ACCOUNT: ${{ secrets.APP_SIGNALS_E2E_TEST_ACC }} - SAMPLE_APP_FRONTEND_SERVICE_JAR: "s3://aws-appsignals-sample-app/main-service.jar" - SAMPLE_APP_REMOTE_SERVICE_JAR: "s3://aws-appsignals-sample-app/remote-service.jar" + SAMPLE_APP_FRONTEND_SERVICE_JAR: ${{ secrets.APP_SIGNALS_E2E_FE_SA_JAR }} + SAMPLE_APP_REMOTE_SERVICE_JAR: ${{ secrets.APP_SIGNALS_E2E_RE_SA_JAR }} APP_SIGNALS_ADOT_JAR: "https://github.com/aws-observability/aws-otel-java-instrumentation/releases/latest/download/aws-opentelemetry-agent.jar" METRIC_NAMESPACE: AppSignals LOG_GROUP_NAME: /aws/appsignals/generic @@ -47,7 +47,6 @@ jobs: else echo APP_SIGNALS_CW_AGENT_RPM="https://amazoncloudwatch-agent-${{ env.AWS_DEFAULT_REGION }}.s3.${{ env.AWS_DEFAULT_REGION }}.amazonaws.com/amazon_linux/amd64/1.300031.0b313/amazon-cloudwatch-agent.rpm" >> $GITHUB_ENV fi - - name: Generate testing id run: echo TESTING_ID="${{ github.run_id }}-${{ github.run_number }}" >> $GITHUB_ENV diff --git a/.github/workflows/appsignals-e2e-eks-test.yml b/.github/workflows/appsignals-e2e-eks-test.yml index 87e5997593..315632f512 100644 --- a/.github/workflows/appsignals-e2e-eks-test.yml +++ b/.github/workflows/appsignals-e2e-eks-test.yml @@ -42,6 +42,24 @@ jobs: with: fetch-depth: 0 + - name: Download enablement script + uses: actions/checkout@v4 + with: + repository: aws-observability/application-signals-demo + ref: main + path: enablement-script + sparse-checkout: | + scripts/eks/appsignals/enable-app-signals.sh + scripts/eks/appsignals/clean-app-signals.sh + sparse-checkout-cone-mode: false + + - name: Remove log group deletion command + if: always() + working-directory: enablement-script/scripts/eks/appsignals + run: | + delete_log_group="aws logs delete-log-group --log-group-name '${{ env.LOG_GROUP_NAME }}' --region \$REGION" + sed -i "s#$delete_log_group##g" clean-app-signals.sh + - uses: actions/setup-java@v4 with: java-version: 17 @@ -90,18 +108,6 @@ jobs: with: terraform_wrapper: false - # Enable App Signals on the test cluster - - name: Pull and unzip enablement script from S3 - working-directory: testing/terraform/eks - run: aws s3 cp ${{ env.ENABLEMENT_SCRIPT_S3_BUCKET }} . && unzip -j onboarding.zip - - - name: Remove log group deletion command - if: always() - working-directory: testing/terraform/eks - run: | - delete_log_group="aws logs delete-log-group --log-group-name '${{ env.LOG_GROUP_NAME }}' --region \$REGION" - sed -i "s#$delete_log_group##g" clean-app-signals.sh - - name: Deploy sample app via terraform and wait for the endpoint to come online id: deploy-sample-app working-directory: testing/terraform/eks @@ -138,7 +144,7 @@ jobs: # after installing App Signals. Attempts to connect will be made for up to 10 minutes if [ $deployment_failed -eq 0 ]; then echo "Installing app signals to the sample app" - ./enable-app-signals.sh \ + ${GITHUB_WORKSPACE}/enablement-script/scripts/eks/appsignals/enable-app-signals.sh \ ${{ inputs.test-cluster-name }} \ ${{ env.AWS_DEFAULT_REGION }} \ ${{ env.SAMPLE_APP_NAMESPACE }} @@ -325,7 +331,7 @@ jobs: - name: Clean Up App Signals if: always() continue-on-error: true - working-directory: testing/terraform/eks + working-directory: enablement-script/scripts/eks/appsignals run: | ./clean-app-signals.sh \ ${{ inputs.test-cluster-name }} \ diff --git a/testing/README.md b/testing/README.md new file mode 100644 index 0000000000..d5e94a6ab3 --- /dev/null +++ b/testing/README.md @@ -0,0 +1,111 @@ +# How to Test E2E Resource Changes +This guide will give a step by step instruction on how to test changes made to E2E testing resources before pushing a PR. +The guide will include the following: +- Setting up IAM roles and an EKS cluster +- Setting up VPC settings and IAM role for EC2 instances +- Buliding sample app images/files and putting them into ECRs/S3 buckets +- Forking a repository and setting up necessary secrets + + +### 1. Create an IAM Role with OIDC Identity Provider +This step is needed to allow Github Action to have access to resources in the AWS account +#### Create an OIDC Provider +- First step is to create an OIDC Identity Provider to allow Github action access to the AWS account resource. Login to AWS, go to the IAM console and click on the Identity Providers tab. +- Click on Add Provider, choose OpenID Connect and type `https://token.actions.githubusercontent.com` in the Provider URL. Click "Get thumbprint". For Audience, use `sts.amazonaws.com`. Finally, click "Add provider" +#### Create an IAM role +- Next, an IAM role needs to be created using the OIDC Identity Provider. Go to the Roles tab and click Create role. +- Choose Web Identity, and choose `token.actions.githubusercontent.com` as the Identity provider, Audience as `sts.amazonaws.com`, and for Github organizations put your github username down. Click next. +- Add the AdministratorAccess policy. Click next. +- Enter your Role name. Click "Create role". +#### Add Additional Permission +- After the role is created, search the role name in the roles tab, click on the role, and go to the Trust relationships tab. Click on "Edit trust policy". +- In the Statement list, add the following item: +`{ + "Sid": "accessToRole", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::root" + }, + "Action": "sts:AssumeRole" + }`. This additional permission is need to allow Github Action to assume roles and have access to the EKS cluster. + +Additional Resource: https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services + +### 2. Create EKS Clusters +The E2E EKS test uses an EKS cluster to deploy the sample apps. +#### Setup Environment with the Appropriate Roles and Permissions. +Note: Make sure to replace with the correct value. +- First, login to the testing account +- Assume the e2e test role by running + - `output=$(aws sts assume-role --role-arn arn:aws:iam:::role/ --role-session-name AWSCLI-Session)` + - `export AWS_ACCESS_KEY_ID=$(echo $output | jq -r .Credentials.AccessKeyId)` + - `export AWS_SECRET_ACCESS_KEY=$(echo $output | jq -r .Credentials.SecretAccessKey)` + - `export AWS_SESSION_TOKEN=$(echo $output | jq -r .Credentials.SessionToken)` +- Run `aws sts get-caller-identity` to check if you are in the correct role +#### Create a new Cluster +Note: Make sure to replace with the correct value (e.g. us-east-1) +- Next, create the cluster by running `eksctl create cluster --name e2e-canary-test --region --zones a,b`. This will take around ~10 minutes. +#### Install AWS Load Balancer Controller Add-on +- Finally, install the AWS Load Balancer Controller add-on by running the following commands. Make sure to replace the `` and `` with the correct value. + ``` + eksctl utils associate-iam-oidc-provider --cluster e2e-canary-test --region --approve + curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.4.7/docs/install/iam_policy.json + aws iam create-policy --policy-name AWSLoadBalancerControllerIAMPolicy --policy-document file://iam_policy.json --region + eksctl create iamserviceaccount --cluster=e2e-canary-test --namespace=kube-system --name=aws-load-balancer-controller --attach-policy-arn=arn:aws:iam:::policy/AWSLoadBalancerControllerIAMPolicy --region --approve + kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v1.5.4/cert-manager.yaml + curl -Lo v2_4_7_full.yaml https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.4.7/v2_4_7_full.yaml + sed -i.bak -e '561,569d' ./v2_4_7_full.yaml + sed -i.bak -e 's|your-cluster-name|e2e-canary-test|' ./v2_4_7_full.yaml + kubectl apply -f v2_4_7_full.yaml + curl -Lo v2_4_7_ingclass.yaml https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.4.7/v2_4_7_ingclass.yaml + kubectl apply -f v2_4_7_ingclass.yaml``` +#### Create a New Cluster and Install the AWS Load Balancer Controller Add-on for All Regions +Repeat the previous two steps for all 5 regions. (us-east-1, us-east-2, eu-west-1, ap-northeast-1, ap-southeast-2) + +Note: If you do not want to test all 5 regions, you can create one for us-east-1, then on the [appsignals-e2e-eks-canary-test.yml](.github/workflows/appsignals-e2e-eks-canary-test.yml) comment out the other regions on line 24 + +### 3. Setting up Environment for EC2 Tests +#### Create IAM Role for EC2 Instance +- Login to AWS, go to the IAM console and click on the Roles tab. Click Create role. +- Choose AWS service, and choose EC2 as the use case. Click Next. +- Choose AmazonS3ReadOnlyAccess, AWSXrayWriteOnlyAccess, and CloudWatchAgentServerPolicy as the permission. +- Type the role name as `APP_SIGNALS_EC2_TEST_ROLE` and click "Create role". + +#### Setting Up Default VPC +- Go to the VPC console and on the routing table for the default VPC, click Edit routes. (The default VPC should have the `-` name if it hasn't been assigned to another VPC before) +- Click add routes, for destination add `0.0.0.0/0`, for target add Internet Gateway and save changes. +- Go to the Security groups tab, find the security group attached to the default VPC, click Edit inbound rules, choose type: All Traffic, Source: custom, and CIDR block: 0.0.0.0/0. Save rules. + +### 4. Building Sample App to ECR +Create two ECR repositories: one for the sample app main service and another for the sample app remote service. +Follow the instructions [here](./sample-apps/README.md) to build the sample app image and upload it to the ECR + +### 5. Building Sample App to S3 Bucket +Create an S3 Bucket to store the .jar files for the sample app main service and sample app remote service. +Follow the instructions under [here](./sample-apps/README.md) to build the sample app .jar and upload it to the bucket + +### 6. Setting up repository +- Go to https://github.com/aws-observability/aws-otel-java-instrumentation and create a fork +- Go to the forked repo and enable action on the Action tab +- Add the following secrets to the repository + - APP_SIGNALS_E2E_TEST_ACC: `` + - E2E_TEST_ROLE_ARN: `arn:aws:iam:::role/` + - APP_SIGNALS_E2E_FE_SA_IMG: `.dkr.ecr.us-east-1.amazonaws.com/` + - APP_SIGNALS_E2E_RE_SA_IMG: `.dkr.ecr.us-east-1.amazonaws.com/` + - APP_SIGNALS_E2E_FE_SA_JAR: s3:/// + - APP_SIGNALS_E2E_RE_SA_JAR: s3:/// + + +### 7. Running the tests +Go to the Actions tab on the forked repository. + +- To run the EKS test, go to `App Signals Enablement - E2E EKS Canary Testing`, and click `Enable Workflow`, then `Run Workflow`. +- To run the EC2 test, go to `App Signals Enablement - E2E EC2 Canary Testing`, and click `Enable Workflow`, then `Run Workflow`. + +If you don't want the canaries to run every 15 minutes on the forked repository, click the `...` button on the top right and choose `Disable Workflow` + +### E2E Testing Resources +- `./.github/workflows/appsignals-e2e-*`: workflow files for running e2e tests +- `./testing/sample-apps/*`: files for building the sample app +- `./testing/validator/*`: files for validating logs/metrics/traces generated by sample app +- `./testing/terraform/*`: files for launching the sample app to EKS cluster or EC2 instances \ No newline at end of file diff --git a/testing/sample-apps/README.md b/testing/sample-apps/README.md index a5c93c74e5..468bcf0912 100644 --- a/testing/sample-apps/README.md +++ b/testing/sample-apps/README.md @@ -10,14 +10,29 @@ Ensure that none of the repositories are currently using the image about to be u To update the image, first push the update to a backup image (or generate a new one), then switch the address on the three repositories to the backup image one by one. Once all three repositories are pointing to the backup image, push the update to the main image and revert the addresses on the repositories back to the original. Be careful to ensure the image names are appropriately stored in secrets. +### Setting up the environment: +1. Run `./.github/scripts/patch.sh` in the repository root. You should have a new folder called `opentelemetry-java-instrumentation` +2. Cd to the new folder, then run `gradle publishToMavenLocal` +3. Run `rm -rf opentelemetry-java-instrumentation` to delete the folder. + ### Steps to update image: -1. Use `ada` commands to autheticate into the testing account -2. Login to ECR Repository: `aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin {REPOSITORY}` +1. Login to the testing account +2. Create a new ECR repository if there's no existing one. +2. Login to ECR Repository: `aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin {REPOSITORY}`. 3. Change repository name in the `build.gradle.kts` file under `testing/sample-apps/springboot` or `testing/sample-apps/sprintboot-remote-service` +4. Change the `tasks.named("jib").enabled` value on the `build.gradle.kts` file from false to true 4. Run `gradle jib` under the respective directory. -## [WIP] EC2 Use Case: Building the JAR Files +## EC2 Use Case: Building the JAR Files To build the JAR files of the sample application, simply `cd` into each application, e.g. `cd testing/sample-apps/springboot`, and run `gradle build`. -This will create a JAR file in the `build/libs/` folder. To update the JAR file in the testing account: -- Use `ada` commands to authenticate into the testing account -- Only after you're sure of your changes and if they do not break the tests running in other repos, use `aws s3api put-object --bucket --body build/libs/.jar --key .jar` to push the JAR to S3 +This will create JAR files in the `build/libs/` folder with the format: +- springboot-*-SNAPSHOT-javadoc.jar +- springboot-*-SNAPSHOT-plain.jar +- springboot-*-SNAPSHOT-sources.jar +- springboot-*-SNAPSHOT.jar. + +To update the JAR file in the testing account: +- Login to the testing account +- Only after you're sure of your changes and if they do not break the tests running in other repos, use `aws s3api put-object --bucket --body build/libs/springboot-*-SNAPSHOT.jar --key .jar` + +Note: Replace * with the version number and `.jar` is the desired name of the .jar file once in the s3 bucket. e.g. `sample-app-main-service.jar`