feat: Added Support for Test Account Management

appveyor-ubuntu.yml

@@ -13,6 +13,8 @@ environment:
   AWS_DEFAULT_REGION: us-east-1
   SAM_CLI_DEV: 1
   NODE_VERSION: "14.17.6"
+  AWS_S3: 'AWS_S3_TESTING'
+  AWS_ECR: 'AWS_ECR_TESTING'
 
   matrix:
 
@@ -24,8 +26,6 @@ environment:
       INSTALL_PY_36_PIP: 1
       INSTALL_PY_38_PIP: 1
       INSTALL_PY_39_PIP: 1
-      AWS_S3: 'AWS_S3_37'
-      AWS_ECR: 'AWS_ECR_37'
       APPVEYOR_CONSOLE_DISABLE_PTY: true
 
     - PYTHON_HOME: "C:\\Python38-x64"
@@ -36,8 +36,6 @@ environment:
       INSTALL_PY_36_PIP: 1
       INSTALL_PY_37_PIP: 1
       INSTALL_PY_39_PIP: 1
-      AWS_S3: 'AWS_S3_38'
-      AWS_ECR: 'AWS_ECR_38'
       APPVEYOR_CONSOLE_DISABLE_PTY: true
 
     - PYTHON_HOME: "C:\\Python39-x64"
@@ -48,8 +46,6 @@ environment:
       INSTALL_PY_36_PIP: 1
       INSTALL_PY_37_PIP: 1
       INSTALL_PY_38_PIP: 1
-      AWS_S3: 'AWS_S3_39'
-      AWS_ECR: 'AWS_ECR_39'
       APPVEYOR_CONSOLE_DISABLE_PTY: true
 
 install:
@@ -85,7 +81,7 @@ install:
   - sh: "sudo apt-get -y install python2.7"
   - sh: "sudo apt-get -y install python3.7"
   - sh: "sudo apt-get -y install python3.8"
-  - sh: "sudo apt-get -y install python3.9"
+  - sh: "sudo apt-get -y install python3.9 python3.9-dev python3.9-venv"
 
   - sh: "which python3.8"
   - sh: "which python3.7"
@@ -104,15 +100,66 @@ install:
   - ps: "If ($env:INSTALL_PY_37_PIP) {python3.7 get-pip.py --user}"
   - ps: "If ($env:INSTALL_PY_36_PIP) {python3.6 get-pip-36.py --user}"
 
+  # update ca-certificates which causes failures with newest golang library
+  - sh: "sudo apt-get install --reinstall ca-certificates"
+
+  # get testing env vars
+  - sh: "sudo apt install -y jq"
+
+  - sh: "python3.9 -m venv .venv_env_vars"
+  - sh: ".venv_env_vars/bin/pip install boto3"
+  - sh: "test_env_var=$(.venv_env_vars/bin/python tests/get_testing_resources.py)"
+  - sh: '
+    if [ $? -ne 0 ]; then
+      echo "get_testing_resources failed. Failed to acquire credentials or test resources.";
+      false;
+    fi
+  '
+
+  - sh: 'export CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID'
+  - sh: 'export CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY'
+  - sh: 'export CI_ACCESS_ROLE_AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN'
+
+  - sh: 'export AWS_ACCESS_KEY_ID=$(echo "$test_env_var" | jq -j ".accessKeyID")'
+  - sh: 'export AWS_SECRET_ACCESS_KEY=$(echo "$test_env_var" | jq -j ".secretAccessKey")'
+  - sh: 'export AWS_SESSION_TOKEN=$(echo "$test_env_var" | jq -j ".sessionToken")'
+  - sh: 'export TASK_TOKEN=$(echo "$test_env_var" | jq -j ".taskToken")'
+  - sh: 'export AWS_S3_TESTING=$(echo "$test_env_var" | jq -j ".TestBucketName")'
+  - sh: 'export AWS_ECR_TESTING=$(echo "$test_env_var" | jq -j ".TestECRURI")'
+  - sh: 'export AWS_KMS_KEY=$(echo "$test_env_var" | jq -j ".TestKMSKeyArn")'
+  - sh: 'export AWS_SIGNING_PROFILE_NAME=$(echo "$test_env_var" | jq -j ".TestSigningProfileName")'
+  - sh: 'export AWS_SIGNING_PROFILE_VERSION_ARN=$(echo "$test_env_var" | jq -j ".TestSigningProfileARN")'
+
   # required for RIE with arm64 in linux
+  - sh: "
+    if [[ -n $BY_CANARY ]] && [[ -n $DOCKER_USER ]] && [[ -n $DOCKER_PASS ]];
+      then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin registry-1.docker.io; 
+    fi"
+  - sh: "
+    if [[ -n $BY_CANARY ]] && [[ -n $DOCKER_USER ]] && [[ -n $DOCKER_PASS ]];
+      then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin; 
+    fi"
   - sh: "docker run --rm --privileged multiarch/qemu-user-static --reset -p yes"
 
-  # update ca-certificates which causes failures with newest golang library
-  - sh: "sudo apt-get install --reinstall ca-certificates"
+
+  # Runs only in Linux, logging Public ECR when running canary and cred is available
+  - sh: "
+    if [[ -n $BY_CANARY ]];
+      then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; 
+    fi"
+
 
 build_script:
       - "python -c \"import sys; print(sys.executable)\""
 
+# Final clean up no matter success or failure
+on_finish:
+  - sh: 'export AWS_ACCESS_KEY_ID=$CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID'
+  - sh: 'export AWS_SECRET_ACCESS_KEY=$CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY'
+  - sh: 'export AWS_SESSION_TOKEN=$CI_ACCESS_ROLE_AWS_SESSION_TOKEN'
+
+  - sh: 'aws stepfunctions send-task-success --task-token "$TASK_TOKEN" --task-output "{}" --region us-west-2'
+
 for:
   # Integ testing build
   - 
@@ -125,12 +172,6 @@ for:
       # Pre-dev Tests
       - "pip install -e \".[pre-dev]\""
 
-      # Runs only in Linux, logging Public ECR when running canary and cred is available
-      - sh: "
-        if [[ -n $BY_CANARY ]];
-          then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; 
-        fi"
-
       - "pip install -e \".[dev]\""
       - sh: "pytest -vv tests/integration/buildcmd"
 
@@ -146,15 +187,6 @@ for:
           configuration: DeployIntegTesting
 
     test_script:
-      # Pre-dev Tests
-      - "pip install -e \".[pre-dev]\""
-
-      # Runs only in Linux, logging Public ECR when running canary and cred is available
-      - sh: "
-        if [[ -n $BY_CANARY ]];
-          then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws;
-        fi"
-
       - "pip install -e \".[dev]\""
       - sh: "pytest -vv tests/integration/delete tests/integration/deploy tests/integration/package tests/integration/sync"
 
@@ -170,12 +202,6 @@ for:
       - "pip install -e \".[pre-dev]\""
       - "pylint --rcfile .pylintrc samcli"
 
-      # Runs only in Linux, logging Public ECR when running canary and cred is available
-      - sh: "
-        if [[ -n $BY_CANARY ]];
-          then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws;
-        fi"
-
       - "pip install -e \".[dev]\""
       - sh: "pytest -vv tests/integration/local"
 
@@ -197,13 +223,7 @@ for:
       - "pylint --rcfile .pylintrc samcli"
       - "mypy setup.py samcli tests"
       - "pytest -n 4 tests/functional"
-
-      # Runs only in Linux, logging Public ECR when running canary and cred is available
-      - sh: "
-        if [[ -n $BY_CANARY ]];
-          then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws;
-        fi"
-
+
       - sh: "pytest tests/integration --ignore=tests/integration/buildcmd --ignore=tests/integration/delete --ignore=tests/integration/deploy --ignore=tests/integration/package --ignore=tests/integration/sync --ignore=tests/integration/local"
       - sh: "pytest -vv tests/regression"
       - sh: "black --check setup.py tests samcli"

appveyor-windows.yml

@@ -16,8 +16,8 @@ environment:
   SAM_CLI_DEV: 1
 
   # Python uses $TMPDIR envvar to find root of tempdir
-  TMPDIR: '%TEMP%'
-  TMP: '%TEMP%'
+  TMPDIR: "%TEMP%"
+  TMP: "%TEMP%"
 
   # MSI Installers only use Py3.7.6. It is sufficient to test with this version here.
   PYTHON_HOME: "C:\\Python37-x64"
@@ -28,8 +28,8 @@ environment:
   HOMEDRIVE: "C:"
   HOMEPATH: 'C:\Users\appveyor'
   NOSE_PARAMETERIZED_NO_WARN: 1
-  AWS_S3: "AWS_S3_37_WIN"
-  AWS_ECR: "AWS_ECR_37"
+  AWS_S3: "AWS_S3_TESTING"
+  AWS_ECR: "AWS_ECR_TESTING"
   APPVEYOR_CONSOLE_DISABLE_PTY: true
 
 init:
@@ -63,6 +63,36 @@ install:
   # Install AWS CLI Globally via pip3
   - "pip install awscli"
 
+  # Switch to Docker Linux containers
+  - ps: Switch-DockerLinux
+
+  # Check for git executable
+  - "git --version"
+
+  # Get testing env vars
+  - ps: "
+  If (Test-Path env:BY_CANARY){
+    python -m virtualenv venv_env_vars;
+    ./venv_env_vars/Scripts/pip install boto3;
+    $test_env_var = ./venv_env_vars/Scripts/python tests/get_testing_resources.py;
+    $test_env_var_json = ConvertFrom-Json $test_env_var;
+
+    $env:CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID = $env:AWS_ACCESS_KEY_ID;
+    $env:CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY = $env:AWS_SECRET_ACCESS_KEY;
+    $env:CI_ACCESS_ROLE_AWS_SESSION_TOKEN = $env:AWS_SESSION_TOKEN;
+
+    $env:AWS_ACCESS_KEY_ID = $test_env_var_json.accessKeyID;
+    $env:AWS_SECRET_ACCESS_KEY = $test_env_var_json.secretAccessKey;
+    $env:AWS_SESSION_TOKEN = $test_env_var_json.sessionToken;
+    $env:TASK_TOKEN = $test_env_var_json.taskToken;
+    $env:AWS_S3_TESTING = $test_env_var_json.TestBucketName;
+    $env:AWS_ECR_TESTING = $test_env_var_json.TestECRURI;
+    $env:AWS_KMS_KEY = $test_env_var_json.TestKMSKeyArn;
+    $env:AWS_SIGNING_PROFILE_NAME = $test_env_var_json.TestSigningProfileName;
+    $env:AWS_SIGNING_PROFILE_VERSION_ARN = $test_env_var_json.TestSigningProfileARN;
+  }"
+
+
   # Create new virtual environment with chosen python version and activate it
   - "python -m virtualenv venv"
   - "venv\\Scripts\\activate"
@@ -74,15 +104,25 @@ install:
   # Install aws cli
   - "pip install awscli"
 
-  # Switch to Docker Linux containers
-  - ps: Switch-DockerLinux
-
-  # Check for git executable
-  - "git --version"
-
   # Echo final Path
   - "echo %PATH%"
 
+  - ps: "
+      If (Test-Path env:BY_CANARY){
+      echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws;
+      }"
+
+# Final clean up no matter success or failure
+on_finish:
+  - ps: "
+  If (Test-Path env:BY_CANARY){
+    $env:AWS_ACCESS_KEY_ID = $env:CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID;
+    $env:AWS_SECRET_ACCESS_KEY = $env:CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY;
+    $env:AWS_SESSION_TOKEN = $env:CI_ACCESS_ROLE_AWS_SESSION_TOKEN;
+    aws stepfunctions send-task-success --task-token \"$env:TASK_TOKEN\" --task-output \"{}\" --region us-west-2;
+  }"
+
+
 for:
   #Integ testing build, regression
   - matrix:
@@ -91,10 +131,6 @@ for:
 
     test_script:
       # Reactivate virtualenv before running tests
-      - ps: "
-          If (Test-Path env:BY_CANARY){
-          echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws;
-          }"
       - "git --version"
       - "venv\\Scripts\\activate"
       - "docker system prune -a -f"
@@ -107,10 +143,6 @@ for:
 
     test_script:
       # Reactivate virtualenv before running tests
-      - ps: "
-          If (Test-Path env:BY_CANARY){
-          echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws;
-          }"
       - "git --version"
       - "venv\\Scripts\\activate"
       - "docker system prune -a -f"
@@ -123,10 +155,6 @@ for:
 
     test_script:
       # Reactivate virtualenv before running tests
-      - ps: "
-          If (Test-Path env:BY_CANARY){
-          echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws;
-          }"
       - "git --version"
       - "venv\\Scripts\\activate"
       - "docker system prune -a -f"
@@ -139,10 +167,6 @@ for:
 
     test_script:
       # Reactivate virtualenv before running tests
-      - ps: "
-          If (Test-Path env:BY_CANARY){
-          echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws;
-          }"
       - "git --version"
       - "venv\\Scripts\\activate"
       - "docker system prune -a -f"

appveyor.yml

@@ -169,9 +169,6 @@ for:
       - ps: "If ($env:INSTALL_PY_37_PIP) {python3.7 get-pip.py --user}"
       - ps: "If ($env:INSTALL_PY_36_PIP) {python3.6 get-pip-36.py --user}"
 
-      # required for RIE with arm64 in linux
-      - sh: "docker run --rm --privileged multiarch/qemu-user-static --reset -p yes"
-
       # update ca-certificates which causes failures with newest golang library
       - sh: "sudo apt-get install --reinstall ca-certificates"
 

tests/get_testing_resources.py

@@ -0,0 +1,59 @@
+"""
+Script for getting test account credentials and managed test account resources.
+The output will be a json string with creds and resource names.
+"""
+import json
+import os
+
+import boto3
+from boto3.session import Session
+from botocore.config import Config
+
+DEFAULT_BOTO_CONFIG = Config(retries={"max_attempts": 10, "mode": "standard"})
+MANAGED_TEST_RESOURCE_STACK_NAME = "managed-test-resources"
+LAMBDA_TIME_OUT = 300
+
+
+def main():
+    env_vars = get_testing_credentials()
+    # Assume testing account credential in order to access managed test resource stack
+    test_session = Session(
+        aws_access_key_id=env_vars["accessKeyID"],
+        aws_secret_access_key=env_vars["secretAccessKey"],
+        aws_session_token=env_vars["sessionToken"],
+    )
+    env_vars.update(get_managed_test_resource_outputs(test_session))
+    print(json.dumps(env_vars))
+
+
+def get_managed_test_resource_outputs(session: Session):
+    """Read output of the managed test resource stack for resource names and arns"""
+    cfn_resource = session.resource("cloudformation", config=DEFAULT_BOTO_CONFIG, region_name="us-east-1")
+    stack = cfn_resource.Stack(MANAGED_TEST_RESOURCE_STACK_NAME)
+    outputs_dict = dict()
+    for output in stack.outputs:
+        outputs_dict[output["OutputKey"]] = output["OutputValue"]
+    return outputs_dict
+
+
+def get_testing_credentials():
+    lambda_arn = os.environ["CREDENTIAL_DISTRIBUTION_LAMBDA_ARN"]
+    # Max attempts to 0 so that boto3 will not invoke multiple times
+    lambda_client = boto3.client(
+        "lambda",
+        config=Config(
+            retries={"max_attempts": 0, "mode": "standard"},
+            connect_timeout=LAMBDA_TIME_OUT + 60,
+            read_timeout=LAMBDA_TIME_OUT + 60,
+        ),
+        region_name="us-west-2",
+    )
+    response = lambda_client.invoke(FunctionName=lambda_arn)
+    payload = json.loads(response["Payload"].read())
+    if response.get("FunctionError"):
+        raise ValueError(f"Failed to get credential. {payload['errorType']}")
+    return payload
+
+
+if __name__ == "__main__":
+    main()