diff --git a/appveyor-ubuntu.yml b/appveyor-ubuntu.yml index 8bb4f7c430..c834ceb6ee 100644 --- a/appveyor-ubuntu.yml +++ b/appveyor-ubuntu.yml @@ -13,6 +13,8 @@ environment: AWS_DEFAULT_REGION: us-east-1 SAM_CLI_DEV: 1 NODE_VERSION: "14.17.6" + AWS_S3: 'AWS_S3_TESTING' + AWS_ECR: 'AWS_ECR_TESTING' matrix: @@ -24,8 +26,6 @@ environment: INSTALL_PY_36_PIP: 1 INSTALL_PY_38_PIP: 1 INSTALL_PY_39_PIP: 1 - AWS_S3: 'AWS_S3_37' - AWS_ECR: 'AWS_ECR_37' APPVEYOR_CONSOLE_DISABLE_PTY: true - PYTHON_HOME: "C:\\Python38-x64" @@ -36,8 +36,6 @@ environment: INSTALL_PY_36_PIP: 1 INSTALL_PY_37_PIP: 1 INSTALL_PY_39_PIP: 1 - AWS_S3: 'AWS_S3_38' - AWS_ECR: 'AWS_ECR_38' APPVEYOR_CONSOLE_DISABLE_PTY: true - PYTHON_HOME: "C:\\Python39-x64" @@ -48,8 +46,6 @@ environment: INSTALL_PY_36_PIP: 1 INSTALL_PY_37_PIP: 1 INSTALL_PY_38_PIP: 1 - AWS_S3: 'AWS_S3_39' - AWS_ECR: 'AWS_ECR_39' APPVEYOR_CONSOLE_DISABLE_PTY: true install: @@ -85,7 +81,7 @@ install: - sh: "sudo apt-get -y install python2.7" - sh: "sudo apt-get -y install python3.7" - sh: "sudo apt-get -y install python3.8" - - sh: "sudo apt-get -y install python3.9" + - sh: "sudo apt-get -y install python3.9 python3.9-dev python3.9-venv" - sh: "which python3.8" - sh: "which python3.7" @@ -104,15 +100,66 @@ install: - ps: "If ($env:INSTALL_PY_37_PIP) {python3.7 get-pip.py --user}" - ps: "If ($env:INSTALL_PY_36_PIP) {python3.6 get-pip-36.py --user}" + # update ca-certificates which causes failures with newest golang library + - sh: "sudo apt-get install --reinstall ca-certificates" + + # get testing env vars + - sh: "sudo apt install -y jq" + + - sh: "python3.9 -m venv .venv_env_vars" + - sh: ".venv_env_vars/bin/pip install boto3" + - sh: "test_env_var=$(.venv_env_vars/bin/python tests/get_testing_resources.py)" + - sh: ' + if [ $? -ne 0 ]; then + echo "get_testing_resources failed. Failed to acquire credentials or test resources."; + false; + fi + ' + + - sh: 'export CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID' + - sh: 'export CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY' + - sh: 'export CI_ACCESS_ROLE_AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN' + + - sh: 'export AWS_ACCESS_KEY_ID=$(echo "$test_env_var" | jq -j ".accessKeyID")' + - sh: 'export AWS_SECRET_ACCESS_KEY=$(echo "$test_env_var" | jq -j ".secretAccessKey")' + - sh: 'export AWS_SESSION_TOKEN=$(echo "$test_env_var" | jq -j ".sessionToken")' + - sh: 'export TASK_TOKEN=$(echo "$test_env_var" | jq -j ".taskToken")' + - sh: 'export AWS_S3_TESTING=$(echo "$test_env_var" | jq -j ".TestBucketName")' + - sh: 'export AWS_ECR_TESTING=$(echo "$test_env_var" | jq -j ".TestECRURI")' + - sh: 'export AWS_KMS_KEY=$(echo "$test_env_var" | jq -j ".TestKMSKeyArn")' + - sh: 'export AWS_SIGNING_PROFILE_NAME=$(echo "$test_env_var" | jq -j ".TestSigningProfileName")' + - sh: 'export AWS_SIGNING_PROFILE_VERSION_ARN=$(echo "$test_env_var" | jq -j ".TestSigningProfileARN")' + # required for RIE with arm64 in linux + - sh: " + if [[ -n $BY_CANARY ]] && [[ -n $DOCKER_USER ]] && [[ -n $DOCKER_PASS ]]; + then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin registry-1.docker.io; + fi" + - sh: " + if [[ -n $BY_CANARY ]] && [[ -n $DOCKER_USER ]] && [[ -n $DOCKER_PASS ]]; + then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin; + fi" - sh: "docker run --rm --privileged multiarch/qemu-user-static --reset -p yes" - # update ca-certificates which causes failures with newest golang library - - sh: "sudo apt-get install --reinstall ca-certificates" + + # Runs only in Linux, logging Public ECR when running canary and cred is available + - sh: " + if [[ -n $BY_CANARY ]]; + then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; + fi" + build_script: - "python -c \"import sys; print(sys.executable)\"" +# Final clean up no matter success or failure +on_finish: + - sh: 'export AWS_ACCESS_KEY_ID=$CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID' + - sh: 'export AWS_SECRET_ACCESS_KEY=$CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY' + - sh: 'export AWS_SESSION_TOKEN=$CI_ACCESS_ROLE_AWS_SESSION_TOKEN' + + - sh: 'aws stepfunctions send-task-success --task-token "$TASK_TOKEN" --task-output "{}" --region us-west-2' + for: # Integ testing build - @@ -125,12 +172,6 @@ for: # Pre-dev Tests - "pip install -e \".[pre-dev]\"" - # Runs only in Linux, logging Public ECR when running canary and cred is available - - sh: " - if [[ -n $BY_CANARY ]]; - then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; - fi" - - "pip install -e \".[dev]\"" - sh: "pytest -vv tests/integration/buildcmd" @@ -146,15 +187,6 @@ for: configuration: DeployIntegTesting test_script: - # Pre-dev Tests - - "pip install -e \".[pre-dev]\"" - - # Runs only in Linux, logging Public ECR when running canary and cred is available - - sh: " - if [[ -n $BY_CANARY ]]; - then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; - fi" - - "pip install -e \".[dev]\"" - sh: "pytest -vv tests/integration/delete tests/integration/deploy tests/integration/package tests/integration/sync" @@ -170,12 +202,6 @@ for: - "pip install -e \".[pre-dev]\"" - "pylint --rcfile .pylintrc samcli" - # Runs only in Linux, logging Public ECR when running canary and cred is available - - sh: " - if [[ -n $BY_CANARY ]]; - then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; - fi" - - "pip install -e \".[dev]\"" - sh: "pytest -vv tests/integration/local" @@ -197,13 +223,7 @@ for: - "pylint --rcfile .pylintrc samcli" - "mypy setup.py samcli tests" - "pytest -n 4 tests/functional" - - # Runs only in Linux, logging Public ECR when running canary and cred is available - - sh: " - if [[ -n $BY_CANARY ]]; - then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; - fi" - + - sh: "pytest tests/integration --ignore=tests/integration/buildcmd --ignore=tests/integration/delete --ignore=tests/integration/deploy --ignore=tests/integration/package --ignore=tests/integration/sync --ignore=tests/integration/local" - sh: "pytest -vv tests/regression" - sh: "black --check setup.py tests samcli" diff --git a/appveyor-windows.yml b/appveyor-windows.yml index 2cf9787664..1bc0dcc5fe 100644 --- a/appveyor-windows.yml +++ b/appveyor-windows.yml @@ -16,8 +16,8 @@ environment: SAM_CLI_DEV: 1 # Python uses $TMPDIR envvar to find root of tempdir - TMPDIR: '%TEMP%' - TMP: '%TEMP%' + TMPDIR: "%TEMP%" + TMP: "%TEMP%" # MSI Installers only use Py3.7.6. It is sufficient to test with this version here. PYTHON_HOME: "C:\\Python37-x64" @@ -28,8 +28,8 @@ environment: HOMEDRIVE: "C:" HOMEPATH: 'C:\Users\appveyor' NOSE_PARAMETERIZED_NO_WARN: 1 - AWS_S3: "AWS_S3_37_WIN" - AWS_ECR: "AWS_ECR_37" + AWS_S3: "AWS_S3_TESTING" + AWS_ECR: "AWS_ECR_TESTING" APPVEYOR_CONSOLE_DISABLE_PTY: true init: @@ -63,6 +63,36 @@ install: # Install AWS CLI Globally via pip3 - "pip install awscli" + # Switch to Docker Linux containers + - ps: Switch-DockerLinux + + # Check for git executable + - "git --version" + + # Get testing env vars + - ps: " + If (Test-Path env:BY_CANARY){ + python -m virtualenv venv_env_vars; + ./venv_env_vars/Scripts/pip install boto3; + $test_env_var = ./venv_env_vars/Scripts/python tests/get_testing_resources.py; + $test_env_var_json = ConvertFrom-Json $test_env_var; + + $env:CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID = $env:AWS_ACCESS_KEY_ID; + $env:CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY = $env:AWS_SECRET_ACCESS_KEY; + $env:CI_ACCESS_ROLE_AWS_SESSION_TOKEN = $env:AWS_SESSION_TOKEN; + + $env:AWS_ACCESS_KEY_ID = $test_env_var_json.accessKeyID; + $env:AWS_SECRET_ACCESS_KEY = $test_env_var_json.secretAccessKey; + $env:AWS_SESSION_TOKEN = $test_env_var_json.sessionToken; + $env:TASK_TOKEN = $test_env_var_json.taskToken; + $env:AWS_S3_TESTING = $test_env_var_json.TestBucketName; + $env:AWS_ECR_TESTING = $test_env_var_json.TestECRURI; + $env:AWS_KMS_KEY = $test_env_var_json.TestKMSKeyArn; + $env:AWS_SIGNING_PROFILE_NAME = $test_env_var_json.TestSigningProfileName; + $env:AWS_SIGNING_PROFILE_VERSION_ARN = $test_env_var_json.TestSigningProfileARN; + }" + + # Create new virtual environment with chosen python version and activate it - "python -m virtualenv venv" - "venv\\Scripts\\activate" @@ -74,15 +104,25 @@ install: # Install aws cli - "pip install awscli" - # Switch to Docker Linux containers - - ps: Switch-DockerLinux - - # Check for git executable - - "git --version" - # Echo final Path - "echo %PATH%" + - ps: " + If (Test-Path env:BY_CANARY){ + echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; + }" + +# Final clean up no matter success or failure +on_finish: + - ps: " + If (Test-Path env:BY_CANARY){ + $env:AWS_ACCESS_KEY_ID = $env:CI_ACCESS_ROLE_AWS_ACCESS_KEY_ID; + $env:AWS_SECRET_ACCESS_KEY = $env:CI_ACCESS_ROLE_AWS_SECRET_ACCESS_KEY; + $env:AWS_SESSION_TOKEN = $env:CI_ACCESS_ROLE_AWS_SESSION_TOKEN; + aws stepfunctions send-task-success --task-token \"$env:TASK_TOKEN\" --task-output \"{}\" --region us-west-2; + }" + + for: #Integ testing build, regression - matrix: @@ -91,10 +131,6 @@ for: test_script: # Reactivate virtualenv before running tests - - ps: " - If (Test-Path env:BY_CANARY){ - echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; - }" - "git --version" - "venv\\Scripts\\activate" - "docker system prune -a -f" @@ -107,10 +143,6 @@ for: test_script: # Reactivate virtualenv before running tests - - ps: " - If (Test-Path env:BY_CANARY){ - echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; - }" - "git --version" - "venv\\Scripts\\activate" - "docker system prune -a -f" @@ -123,10 +155,6 @@ for: test_script: # Reactivate virtualenv before running tests - - ps: " - If (Test-Path env:BY_CANARY){ - echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; - }" - "git --version" - "venv\\Scripts\\activate" - "docker system prune -a -f" @@ -139,10 +167,6 @@ for: test_script: # Reactivate virtualenv before running tests - - ps: " - If (Test-Path env:BY_CANARY){ - echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; - }" - "git --version" - "venv\\Scripts\\activate" - "docker system prune -a -f" diff --git a/appveyor.yml b/appveyor.yml index d2fd354fbf..7400229f05 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -169,9 +169,6 @@ for: - ps: "If ($env:INSTALL_PY_37_PIP) {python3.7 get-pip.py --user}" - ps: "If ($env:INSTALL_PY_36_PIP) {python3.6 get-pip-36.py --user}" - # required for RIE with arm64 in linux - - sh: "docker run --rm --privileged multiarch/qemu-user-static --reset -p yes" - # update ca-certificates which causes failures with newest golang library - sh: "sudo apt-get install --reinstall ca-certificates" diff --git a/tests/get_testing_resources.py b/tests/get_testing_resources.py new file mode 100644 index 0000000000..2d19a9cd7d --- /dev/null +++ b/tests/get_testing_resources.py @@ -0,0 +1,59 @@ +""" +Script for getting test account credentials and managed test account resources. +The output will be a json string with creds and resource names. +""" +import json +import os + +import boto3 +from boto3.session import Session +from botocore.config import Config + +DEFAULT_BOTO_CONFIG = Config(retries={"max_attempts": 10, "mode": "standard"}) +MANAGED_TEST_RESOURCE_STACK_NAME = "managed-test-resources" +LAMBDA_TIME_OUT = 300 + + +def main(): + env_vars = get_testing_credentials() + # Assume testing account credential in order to access managed test resource stack + test_session = Session( + aws_access_key_id=env_vars["accessKeyID"], + aws_secret_access_key=env_vars["secretAccessKey"], + aws_session_token=env_vars["sessionToken"], + ) + env_vars.update(get_managed_test_resource_outputs(test_session)) + print(json.dumps(env_vars)) + + +def get_managed_test_resource_outputs(session: Session): + """Read output of the managed test resource stack for resource names and arns""" + cfn_resource = session.resource("cloudformation", config=DEFAULT_BOTO_CONFIG, region_name="us-east-1") + stack = cfn_resource.Stack(MANAGED_TEST_RESOURCE_STACK_NAME) + outputs_dict = dict() + for output in stack.outputs: + outputs_dict[output["OutputKey"]] = output["OutputValue"] + return outputs_dict + + +def get_testing_credentials(): + lambda_arn = os.environ["CREDENTIAL_DISTRIBUTION_LAMBDA_ARN"] + # Max attempts to 0 so that boto3 will not invoke multiple times + lambda_client = boto3.client( + "lambda", + config=Config( + retries={"max_attempts": 0, "mode": "standard"}, + connect_timeout=LAMBDA_TIME_OUT + 60, + read_timeout=LAMBDA_TIME_OUT + 60, + ), + region_name="us-west-2", + ) + response = lambda_client.invoke(FunctionName=lambda_arn) + payload = json.loads(response["Payload"].read()) + if response.get("FunctionError"): + raise ValueError(f"Failed to get credential. {payload['errorType']}") + return payload + + +if __name__ == "__main__": + main()