From 22e20dbec40c2b0022aa7bc40d696474f9976fb1 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 5 Jan 2022 03:09:45 -0800 Subject: [PATCH 01/10] fix flaky warm containers testing --- .../local/start_api/start_api_integ_base.py | 22 +++++++++++++++-- .../start_lambda_api_integ_base.py | 24 ++++++++++++++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/tests/integration/local/start_api/start_api_integ_base.py b/tests/integration/local/start_api/start_api_integ_base.py index 5575a11051..ec36b63db3 100644 --- a/tests/integration/local/start_api/start_api_integ_base.py +++ b/tests/integration/local/start_api/start_api_integ_base.py @@ -1,9 +1,10 @@ +import selectors import shutil import uuid from typing import List, Optional, Dict from unittest import TestCase, skipIf import threading -from subprocess import Popen +from subprocess import Popen, PIPE import time import os import random @@ -80,7 +81,24 @@ def start_api(cls): for image in cls.invoke_image: command_list += ["--invoke-image", image] - cls.start_api_process = Popen(command_list) + cls.start_api_process = Popen(command_list, stdout=PIPE, stderr=PIPE) + + sel = selectors.DefaultSelector() + sel.register(cls.start_api_process.stdout, selectors.EVENT_READ) + sel.register(cls.start_api_process.stderr, selectors.EVENT_READ) + + while True: + should_start = False + for key, _ in sel.select(): + data = key.fileobj.readline().decode() + if not data: + break + if "(Press CTRL+C to quit)" in data: + should_start = True + break + if should_start: + break + # we need to wait some time for start-api to start, hence the sleep time.sleep(5) diff --git a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py index 24229ff601..81c2f2ac8d 100644 --- a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py +++ b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py @@ -1,10 +1,11 @@ +import selectors import shutil import signal import uuid from typing import Optional, Dict, List from unittest import TestCase, skipIf import threading -from subprocess import Popen +from subprocess import Popen, PIPE import time import os import random @@ -88,9 +89,26 @@ def start_lambda(cls, wait_time=5): for image in cls.invoke_image: command_list += ["--invoke-image", image] - cls.start_lambda_process = Popen(command_list) + cls.start_lambda_process = Popen(command_list, stdout=PIPE, stderr=PIPE) + + sel = selectors.DefaultSelector() + sel.register(cls.start_lambda_process.stdout, selectors.EVENT_READ) + sel.register(cls.start_lambda_process.stderr, selectors.EVENT_READ) + + while True: + should_start = False + for key, _ in sel.select(): + data = key.fileobj.readline().decode() + if not data: + break + if "(Press CTRL+C to quit)" in data: + should_start = True + break + if should_start: + break + # we need to wait some time for start-lambda to start, hence the sleep - time.sleep(wait_time) + # time.sleep(wait_time) @classmethod def _make_parameter_override_arg(self, overrides): From 32d3a5f4f095bbbee9149dbdeb33eab760eec821 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 5 Jan 2022 03:50:53 -0800 Subject: [PATCH 02/10] use py3.7 instead of 3.8, as it is not available in testing windows host --- ...k_v1_synthesized_template_level3_nested_zip_functions.json | 2 +- ...ynthesized_template_level3_nested_zip_functions_after.json | 2 +- .../infra/cdk/cdk_v1_synthesized_template_zip_functions.json | 4 ++-- .../cdk/cdk_v1_synthesized_template_zip_functions_after.json | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_level3_nested_zip_functions.json b/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_level3_nested_zip_functions.json index d21ff15201..4bdebf1d5c 100644 --- a/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_level3_nested_zip_functions.json +++ b/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_level3_nested_zip_functions.json @@ -52,7 +52,7 @@ "aws:asset:path": "before/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe6948", "aws:asset:is-bundled": false, "aws:asset:property": "Content", - "BuildMethod": "python3.8" + "BuildMethod": "python3.7" } }, "RandomCitiesFunctionServiceRole4EFB1CF5": { diff --git a/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_level3_nested_zip_functions_after.json b/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_level3_nested_zip_functions_after.json index dcbf65c589..a74eaccfd4 100644 --- a/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_level3_nested_zip_functions_after.json +++ b/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_level3_nested_zip_functions_after.json @@ -52,7 +52,7 @@ "aws:asset:path": "after/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe6948", "aws:asset:is-bundled": false, "aws:asset:property": "Content", - "BuildMethod": "python3.8" + "BuildMethod": "python3.7" } }, "RandomCitiesFunctionServiceRole4EFB1CF5": { diff --git a/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_zip_functions.json b/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_zip_functions.json index 8a96dd0f5d..9b8144d231 100644 --- a/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_zip_functions.json +++ b/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_zip_functions.json @@ -52,7 +52,7 @@ "aws:asset:path": "before/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe6948", "aws:asset:is-bundled": false, "aws:asset:property": "Content", - "BuildMethod": "python3.8" + "BuildMethod": "python3.7" } }, "RandomCitiesFunctionServiceRole4EFB1CF5": { @@ -142,7 +142,7 @@ "Ref": "CountriesLibLayer259A7AF1" } ], - "Runtime": "python3.8" + "Runtime": "python3.7" }, "DependsOn": [ "RandomCitiesFunctionServiceRole4EFB1CF5" diff --git a/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_zip_functions_after.json b/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_zip_functions_after.json index b0c07e1a24..ca59fdf28f 100644 --- a/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_zip_functions_after.json +++ b/tests/integration/testdata/sync/infra/cdk/cdk_v1_synthesized_template_zip_functions_after.json @@ -52,7 +52,7 @@ "aws:asset:path": "after/asset.b998895901bf33127f2c9dce715854f8b35aa73fb7eb5245ba9721580bbe6948", "aws:asset:is-bundled": false, "aws:asset:property": "Content", - "BuildMethod": "python3.8" + "BuildMethod": "python3.7" } }, "RandomCitiesFunctionServiceRole4EFB1CF5": { @@ -142,7 +142,7 @@ "Ref": "CountriesLibLayer259A7AF1" } ], - "Runtime": "python3.8" + "Runtime": "python3.7" }, "DependsOn": [ "RandomCitiesFunctionServiceRole4EFB1CF5" From 93f55e5e5bd2366b3f74ab73b8c7d3317f60b844 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 5 Jan 2022 04:18:32 -0800 Subject: [PATCH 03/10] selectors are not working in windows --- .../local/start_api/start_api_integ_base.py | 21 +++---------------- .../start_lambda_api_integ_base.py | 20 +++--------------- 2 files changed, 6 insertions(+), 35 deletions(-) diff --git a/tests/integration/local/start_api/start_api_integ_base.py b/tests/integration/local/start_api/start_api_integ_base.py index ec36b63db3..784516be48 100644 --- a/tests/integration/local/start_api/start_api_integ_base.py +++ b/tests/integration/local/start_api/start_api_integ_base.py @@ -1,4 +1,3 @@ -import selectors import shutil import uuid from typing import List, Optional, Dict @@ -81,27 +80,13 @@ def start_api(cls): for image in cls.invoke_image: command_list += ["--invoke-image", image] - cls.start_api_process = Popen(command_list, stdout=PIPE, stderr=PIPE) - - sel = selectors.DefaultSelector() - sel.register(cls.start_api_process.stdout, selectors.EVENT_READ) - sel.register(cls.start_api_process.stderr, selectors.EVENT_READ) + cls.start_api_process = Popen(command_list, stderr=PIPE) while True: - should_start = False - for key, _ in sel.select(): - data = key.fileobj.readline().decode() - if not data: - break - if "(Press CTRL+C to quit)" in data: - should_start = True - break - if should_start: + line = cls.start_api_process.stderr.readline() + if "(Press CTRL+C to quit)" in str(line): break - # we need to wait some time for start-api to start, hence the sleep - time.sleep(5) - @classmethod def _make_parameter_override_arg(self, overrides): return " ".join(["ParameterKey={},ParameterValue={}".format(key, value) for key, value in overrides.items()]) diff --git a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py index 81c2f2ac8d..a9b7199f6c 100644 --- a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py +++ b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py @@ -89,27 +89,13 @@ def start_lambda(cls, wait_time=5): for image in cls.invoke_image: command_list += ["--invoke-image", image] - cls.start_lambda_process = Popen(command_list, stdout=PIPE, stderr=PIPE) - - sel = selectors.DefaultSelector() - sel.register(cls.start_lambda_process.stdout, selectors.EVENT_READ) - sel.register(cls.start_lambda_process.stderr, selectors.EVENT_READ) + cls.start_lambda_process = Popen(command_list, stderr=PIPE) while True: - should_start = False - for key, _ in sel.select(): - data = key.fileobj.readline().decode() - if not data: - break - if "(Press CTRL+C to quit)" in data: - should_start = True - break - if should_start: + line = cls.start_lambda_process.stderr.readline() + if "(Press CTRL+C to quit)" in str(line): break - # we need to wait some time for start-lambda to start, hence the sleep - # time.sleep(wait_time) - @classmethod def _make_parameter_override_arg(self, overrides): return " ".join(["ParameterKey={},ParameterValue={}".format(key, value) for key, value in overrides.items()]) From 1244fa664ef2fc8ce5b7269ca2cfb88bb7f4f256 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 5 Jan 2022 06:35:43 -0800 Subject: [PATCH 04/10] ruby 2.5 is not supported, use 2.7 instead --- appveyor-ubuntu.yml | 2 +- appveyor.yml | 2 +- tests/integration/buildcmd/test_build_cmd.py | 22 +++++++++---------- tests/integration/sync/test_sync_command.py | 15 ++++++++++++- .../sync/infra/after/Ruby/function/Gemfile | 2 +- .../sync/infra/after/Ruby/layer/Gemfile | 2 +- .../sync/infra/before/Ruby/function/Gemfile | 2 +- .../sync/infra/before/Ruby/layer/Gemfile | 2 +- .../sync/infra/template-python-after.yaml | 6 ++--- .../sync/infra/template-python-before.yaml | 6 ++--- .../sar/ffmpeg-lambda-layer-template.yaml | 2 +- 11 files changed, 38 insertions(+), 25 deletions(-) diff --git a/appveyor-ubuntu.yml b/appveyor-ubuntu.yml index eae2e2768d..277293bdec 100644 --- a/appveyor-ubuntu.yml +++ b/appveyor-ubuntu.yml @@ -74,7 +74,7 @@ install: - sh: "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" - sh: "PATH=$JAVA_HOME/bin:$PATH" - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate" - - sh: "rvm use 2.5" + - sh: "rvm use 2.7" - sh: "docker --version" # Install latest gradle diff --git a/appveyor.yml b/appveyor.yml index 7e9515aaaf..6b47e0b6c9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -137,7 +137,7 @@ for: - sh: "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" - sh: "PATH=$JAVA_HOME/bin:$PATH" - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate" - - sh: "rvm use 2.5" + - sh: "rvm use 2.7.2" - sh: "docker --version" # Install latest gradle diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index ad7485495e..bca3297b33 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -395,13 +395,13 @@ def test_building_default_package_json(self, runtime, use_container, architectur "Skip build tests on windows when running in CI unless overridden", ) class TestBuildCommand_RubyFunctions(BuildIntegRubyBase): - @parameterized.expand(["ruby2.5", "ruby2.7"]) + @parameterized.expand(["ruby2.7"]) @pytest.mark.flaky(reruns=3) @skipIf(SKIP_DOCKER_TESTS or SKIP_DOCKER_BUILD, SKIP_DOCKER_MESSAGE) def test_building_ruby_in_container(self, runtime): self._test_with_default_gemfile(runtime, "use_container", "Ruby", self.test_data_path) - @parameterized.expand(["ruby2.5", "ruby2.7"]) + @parameterized.expand(["ruby2.7"]) @pytest.mark.flaky(reruns=3) def test_building_ruby_in_process(self, runtime): self._test_with_default_gemfile(runtime, False, "Ruby", self.test_data_path) @@ -414,13 +414,13 @@ def test_building_ruby_in_process(self, runtime): class TestBuildCommand_RubyFunctions_With_Architecture(BuildIntegRubyBase): template = "template_with_architecture.yaml" - @parameterized.expand(["ruby2.5", "ruby2.7", ("ruby2.7", "arm64")]) + @parameterized.expand(["ruby2.7", ("ruby2.7", "arm64")]) @pytest.mark.flaky(reruns=3) @skipIf(SKIP_DOCKER_TESTS or SKIP_DOCKER_BUILD, SKIP_DOCKER_MESSAGE) def test_building_ruby_in_container_with_specified_architecture(self, runtime, architecture="x86_64"): self._test_with_default_gemfile(runtime, "use_container", "Ruby", self.test_data_path, architecture) - @parameterized.expand(["ruby2.5", "ruby2.7", ("ruby2.7", "arm64")]) + @parameterized.expand(["ruby2.7", ("ruby2.7", "arm64")]) @pytest.mark.flaky(reruns=3) def test_building_ruby_in_process_with_specified_architecture(self, runtime, architecture="x86_64"): self._test_with_default_gemfile(runtime, False, "Ruby", self.test_data_path, architecture) @@ -436,7 +436,7 @@ class TestBuildCommand_RubyFunctionsWithGemfileInTheRoot(BuildIntegRubyBase): This doesn't apply to containerized build, since it copies only the function folder to the container """ - @parameterized.expand([("ruby2.5"), ("ruby2.7")]) + @parameterized.expand([("ruby2.7")]) @pytest.mark.flaky(reruns=3) def test_building_ruby_in_process_with_root_gemfile(self, runtime): self._prepare_application_environment() @@ -1347,12 +1347,12 @@ class TestBuildWithDedupBuilds(DedupBuildIntegBase): (False, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"), (False, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"), (False, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"), - (False, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"), + (False, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.7"), # container (True, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"), (True, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"), (True, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"), - (True, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"), + (True, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.7"), ] ) @pytest.mark.flaky(reruns=3) @@ -1470,12 +1470,12 @@ class TestBuildWithCacheBuilds(CachedBuildIntegBase): (False, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"), (False, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"), (False, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"), - (False, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"), + (False, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.7"), # container (True, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"), (True, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"), (True, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"), - (True, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"), + (True, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.7"), ] ) @pytest.mark.flaky(reruns=3) @@ -1601,12 +1601,12 @@ class TestParallelBuilds(DedupBuildIntegBase): (False, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"), (False, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"), (False, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"), - (False, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"), + (False, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.7"), # container (True, "Java/gradlew", "aws.example.Hello::myHandler", "aws.example.SecondFunction::myHandler", "java8"), (True, "Node", "main.lambdaHandler", "main.secondLambdaHandler", "nodejs14.x"), (True, "Python", "main.first_function_handler", "main.second_function_handler", "python3.9"), - (True, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.5"), + (True, "Ruby", "app.lambda_handler", "app.second_lambda_handler", "ruby2.7"), ] ) @pytest.mark.flaky(reruns=3) diff --git a/tests/integration/sync/test_sync_command.py b/tests/integration/sync/test_sync_command.py index 19492de598..d9a6448d65 100644 --- a/tests/integration/sync/test_sync_command.py +++ b/tests/integration/sync/test_sync_command.py @@ -1,4 +1,5 @@ import os +import platform import logging import json @@ -29,6 +30,7 @@ # Deploy tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo. # This is to restrict package tests to run outside of CI/CD, when the branch is not master or tests are not run by Canary SKIP_SYNC_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY +IS_WINDOWS = platform.system().lower() == "windows" CFN_SLEEP = 3 API_SLEEP = 5 SFN_SLEEP = 10 @@ -71,8 +73,19 @@ def tearDown(self): cfn_client.delete_stack(StackName=stack_name) super().tearDown() - @parameterized.expand(["ruby", "python"]) + @skipIf( + IS_WINDOWS, + "Skip sync ruby tests in windows", + ) + @parameterized.expand(["ruby"]) + def test_sync_infra_ruby(self, runtime): + self._test_sync_infra(runtime) + + @parameterized.expand(["python"]) def test_sync_infra(self, runtime): + self._test_sync_infra(runtime) + + def _test_sync_infra(self, runtime): template_before = f"infra/template-{runtime}-before.yaml" template_path = str(self.test_data_path.joinpath(template_before)) stack_name = self._method_to_stack_name(self.id()) diff --git a/tests/integration/testdata/sync/infra/after/Ruby/function/Gemfile b/tests/integration/testdata/sync/infra/after/Ruby/function/Gemfile index f363756be8..4de23af04b 100644 --- a/tests/integration/testdata/sync/infra/after/Ruby/function/Gemfile +++ b/tests/integration/testdata/sync/infra/after/Ruby/function/Gemfile @@ -2,4 +2,4 @@ source "https://rubygems.org" gem "ruby-statistics" -ruby '~> 2.7.5' +ruby '~> 2.7' diff --git a/tests/integration/testdata/sync/infra/after/Ruby/layer/Gemfile b/tests/integration/testdata/sync/infra/after/Ruby/layer/Gemfile index 93f90933be..d7bdb01969 100644 --- a/tests/integration/testdata/sync/infra/after/Ruby/layer/Gemfile +++ b/tests/integration/testdata/sync/infra/after/Ruby/layer/Gemfile @@ -1,3 +1,3 @@ source "https://rubygems.org" -ruby '~> 2.7.5' +ruby '~> 2.7' diff --git a/tests/integration/testdata/sync/infra/before/Ruby/function/Gemfile b/tests/integration/testdata/sync/infra/before/Ruby/function/Gemfile index f363756be8..4de23af04b 100644 --- a/tests/integration/testdata/sync/infra/before/Ruby/function/Gemfile +++ b/tests/integration/testdata/sync/infra/before/Ruby/function/Gemfile @@ -2,4 +2,4 @@ source "https://rubygems.org" gem "ruby-statistics" -ruby '~> 2.7.5' +ruby '~> 2.7' diff --git a/tests/integration/testdata/sync/infra/before/Ruby/layer/Gemfile b/tests/integration/testdata/sync/infra/before/Ruby/layer/Gemfile index 93f90933be..d7bdb01969 100644 --- a/tests/integration/testdata/sync/infra/before/Ruby/layer/Gemfile +++ b/tests/integration/testdata/sync/infra/before/Ruby/layer/Gemfile @@ -1,3 +1,3 @@ source "https://rubygems.org" -ruby '~> 2.7.5' +ruby '~> 2.7' diff --git a/tests/integration/testdata/sync/infra/template-python-after.yaml b/tests/integration/testdata/sync/infra/template-python-after.yaml index 869d9b01e6..a812bde9cb 100644 --- a/tests/integration/testdata/sync/infra/template-python-after.yaml +++ b/tests/integration/testdata/sync/infra/template-python-after.yaml @@ -11,7 +11,7 @@ Resources: Properties: CodeUri: after/Python/function/ Handler: app.lambda_handler - Runtime: python3.8 + Runtime: python3.7 Layers: - Ref: HelloWorldLayer Tracing: Active @@ -35,9 +35,9 @@ Resources: Description: Hello World Layer ContentUri: after/Python/layer/ CompatibleRuntimes: - - python3.8 + - python3.7 Metadata: - BuildMethod: python3.8 + BuildMethod: python3.7 HelloStepFunction: Type: AWS::Serverless::StateMachine diff --git a/tests/integration/testdata/sync/infra/template-python-before.yaml b/tests/integration/testdata/sync/infra/template-python-before.yaml index 8a4bed8730..db1e1ef80c 100644 --- a/tests/integration/testdata/sync/infra/template-python-before.yaml +++ b/tests/integration/testdata/sync/infra/template-python-before.yaml @@ -11,7 +11,7 @@ Resources: Properties: CodeUri: before/Python/function/ Handler: app.lambda_handler - Runtime: python3.8 + Runtime: python3.7 Layers: - Ref: HelloWorldLayer Tracing: Active @@ -35,9 +35,9 @@ Resources: Description: Hello World Layer ContentUri: before/Python/layer/ CompatibleRuntimes: - - python3.8 + - python3.7 Metadata: - BuildMethod: python3.8 + BuildMethod: python3.7 HelloStepFunction: Type: AWS::Serverless::StateMachine diff --git a/tests/smoke/templates/sar/ffmpeg-lambda-layer-template.yaml b/tests/smoke/templates/sar/ffmpeg-lambda-layer-template.yaml index e9b1afbf34..03d6c2fff8 100644 --- a/tests/smoke/templates/sar/ffmpeg-lambda-layer-template.yaml +++ b/tests/smoke/templates/sar/ffmpeg-lambda-layer-template.yaml @@ -13,7 +13,7 @@ Resources: CompatibleRuntimes: - nodejs10.x - python3.6 - - ruby2.5 + - ruby2.7 - java8 - go1.x ContentUri: From faa6123c66d4cdc18279834561584bc850bccfc6 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 5 Jan 2022 11:13:37 -0800 Subject: [PATCH 05/10] fix the logic to wait till apis are running in windows --- .../local/start_api/start_api_integ_base.py | 10 ++++++++++ .../local/start_lambda/start_lambda_api_integ_base.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/tests/integration/local/start_api/start_api_integ_base.py b/tests/integration/local/start_api/start_api_integ_base.py index 784516be48..f062ab09f9 100644 --- a/tests/integration/local/start_api/start_api_integ_base.py +++ b/tests/integration/local/start_api/start_api_integ_base.py @@ -87,6 +87,15 @@ def start_api(cls): if "(Press CTRL+C to quit)" in str(line): break + cls.stop_reading_thread = False + + def read_sub_process_stderr(): + while not cls.stop_reading_thread: + cls.start_api_process.stderr.readline() + + cls.read_threading = threading.Thread(target=read_sub_process_stderr) + cls.read_threading.start() + @classmethod def _make_parameter_override_arg(self, overrides): return " ".join(["ParameterKey={},ParameterValue={}".format(key, value) for key, value in overrides.items()]) @@ -95,6 +104,7 @@ def _make_parameter_override_arg(self, overrides): def tearDownClass(cls): # After all the tests run, we need to kill the start-api process. cls.start_api_process.kill() + cls.stop_reading_thread = True @staticmethod def random_port(): diff --git a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py index a9b7199f6c..e69ebefd74 100644 --- a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py +++ b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py @@ -96,6 +96,15 @@ def start_lambda(cls, wait_time=5): if "(Press CTRL+C to quit)" in str(line): break + cls.stop_reading_thread = False + + def read_sub_process_stderr(): + while not cls.stop_reading_thread: + cls.start_lambda_process.stderr.readline() + + cls.read_threading = threading.Thread(target=read_sub_process_stderr) + cls.read_threading.start() + @classmethod def _make_parameter_override_arg(self, overrides): return " ".join(["ParameterKey={},ParameterValue={}".format(key, value) for key, value in overrides.items()]) @@ -104,6 +113,7 @@ def _make_parameter_override_arg(self, overrides): def tearDownClass(cls): # After all the tests run, we need to kill the start_lambda process. cls.start_lambda_process.kill() + cls.stop_reading_thread = True @staticmethod def random_port(): From ffba9b73d2b6d961aa4da3a0d43a765540f9ff7f Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 5 Jan 2022 11:46:45 -0800 Subject: [PATCH 06/10] add auto generated S3 Prefix --- .../integration/delete/test_delete_command.py | 8 ++- .../integration/deploy/test_deploy_command.py | 61 ++++++++++--------- .../integration/package/package_integ_base.py | 1 + .../package/test_package_command_image.py | 2 +- .../package/test_package_command_zip.py | 50 ++++++++------- tests/integration/sync/test_sync_command.py | 14 ++--- 6 files changed, 74 insertions(+), 62 deletions(-) diff --git a/tests/integration/delete/test_delete_command.py b/tests/integration/delete/test_delete_command.py index 8c0416ee5d..c6d07107ba 100644 --- a/tests/integration/delete/test_delete_command.py +++ b/tests/integration/delete/test_delete_command.py @@ -230,6 +230,7 @@ def test_delete_no_prompts_no_s3_prefix_zip(self, template_file): stack_name=stack_name, capabilities="CAPABILITY_IAM", s3_bucket=self.bucket_name, + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", @@ -271,6 +272,7 @@ def test_delete_no_prompts_no_s3_prefix_image(self, template_file): capabilities="CAPABILITY_IAM", image_repository=self.ecr_repo_name, s3_bucket=self.bucket_name, + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", @@ -310,7 +312,7 @@ def test_delete_nested_stacks(self, template_file): # Note(xinhol): --capabilities does not allow passing multiple, we need to fix it # here we use samconfig-deep-nested.toml as a workaround config_file=self.test_data_path.joinpath("samconfig-deep-nested.toml"), - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, @@ -404,6 +406,7 @@ def test_delete_guided_no_stack_name_no_region(self, template_file): stack_name=stack_name, capabilities="CAPABILITY_IAM", s3_bucket=self.bucket_name, + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", @@ -440,6 +443,7 @@ def test_delete_guided_ecr_repository_present(self, template_file): stack_name=stack_name, capabilities="CAPABILITY_IAM", s3_bucket=self.bucket_name, + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", @@ -479,6 +483,7 @@ def test_delete_guided_no_s3_prefix_image(self, template_file): capabilities="CAPABILITY_IAM", image_repository=self.ecr_repo_name, s3_bucket=self.bucket_name, + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", @@ -517,6 +522,7 @@ def test_delete_guided_retain_s3_artifact(self, template_file): stack_name=stack_name, capabilities="CAPABILITY_IAM", s3_bucket=self.bucket_name, + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", diff --git a/tests/integration/deploy/test_deploy_command.py b/tests/integration/deploy/test_deploy_command.py index 3ed0caa606..f4992ea955 100644 --- a/tests/integration/deploy/test_deploy_command.py +++ b/tests/integration/deploy/test_deploy_command.py @@ -96,7 +96,7 @@ def test_package_and_deploy_no_s3_bucket_all_args(self, template_file): template_file=output_template_file.name, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, @@ -114,7 +114,7 @@ def test_package_and_deploy_no_s3_bucket_all_args(self, template_file): template_file=output_template_file.name, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", @@ -143,7 +143,7 @@ def test_no_package_and_deploy_with_s3_bucket_all_args(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, @@ -176,7 +176,7 @@ def test_no_package_and_deploy_image_repository(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, image_repository=self.ecr_repo_name, force_upload=True, @@ -216,7 +216,7 @@ def test_no_package_and_deploy_with_s3_bucket_all_args_image_repositories(self, template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, image_repositories=f"{resource_id}={self.ecr_repo_name}", force_upload=True, @@ -250,7 +250,7 @@ def test_no_package_and_deploy_resolve_image_repos(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, @@ -277,7 +277,7 @@ def test_no_package_and_deploy_with_s3_bucket_and_no_confirm_changeset(self, tem template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, @@ -306,7 +306,7 @@ def test_deploy_no_redeploy_on_same_built_artifacts(self, template_file): deploy_command_list = self.get_deploy_command_list( stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, @@ -342,7 +342,7 @@ def test_no_package_and_deploy_with_s3_bucket_all_args_confirm_changeset(self, t template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, @@ -367,7 +367,7 @@ def test_deploy_without_s3_bucket(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", @@ -397,7 +397,7 @@ def test_deploy_without_stack_name(self, template_file): deploy_command_list = self.get_deploy_command_list( template_file=template_path, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", @@ -420,7 +420,7 @@ def test_deploy_without_capabilities(self, template_file): deploy_command_list = self.get_deploy_command_list( template_file=template_path, stack_name=stack_name, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", @@ -439,7 +439,7 @@ def test_deploy_without_template_file(self): # Package and Deploy in one go without confirming change set. deploy_command_list = self.get_deploy_command_list( stack_name=stack_name, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", @@ -465,7 +465,7 @@ def test_deploy_with_s3_bucket_switch_region(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.bucket_name, force_upload=True, notification_arns=self.sns_arn, @@ -485,7 +485,7 @@ def test_deploy_with_s3_bucket_switch_region(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.bucket_name, force_upload=True, notification_arns=self.sns_arn, @@ -521,7 +521,7 @@ def test_deploy_twice_with_no_fail_on_empty_changeset(self, template_file): "template_file": template_path, "stack_name": stack_name, "capabilities": "CAPABILITY_IAM", - "s3_prefix": "integ_deploy", + "s3_prefix": self.s3_prefix, "s3_bucket": self.bucket_name, "force_upload": True, "notification_arns": self.sns_arn, @@ -561,7 +561,7 @@ def test_deploy_twice_with_fail_on_empty_changeset(self, template_file): "template_file": template_path, "stack_name": stack_name, "capabilities": "CAPABILITY_IAM", - "s3_prefix": "integ_deploy", + "s3_prefix": self.s3_prefix, "s3_bucket": self.bucket_name, "force_upload": True, "notification_arns": self.sns_arn, @@ -780,6 +780,7 @@ def test_deploy_with_no_s3_bucket_set_resolve_s3(self, template_file): kms_key_id=self.kms_key, tags="integ=true clarity=yes foo_bar=baz", resolve_s3=True, + s3_prefix=self.s3_prefix, ) deploy_process_execute = run_command(deploy_command_list) @@ -807,7 +808,7 @@ def test_deploy_with_valid_config_tags_list(self, template_file, config_file): template_file=template_path, stack_name=stack_name, config_file=config_path, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, capabilities="CAPABILITY_IAM", ) @@ -826,7 +827,7 @@ def test_deploy_with_valid_config_tags_string(self, template_file, config_file): template_file=template_path, stack_name=stack_name, config_file=config_path, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, capabilities="CAPABILITY_IAM", ) @@ -868,7 +869,7 @@ def test_deploy_with_code_signing_params(self, should_sign, should_enforce, will template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, @@ -952,7 +953,7 @@ def test_deploy_nested_stacks(self, template_file): # Note(xinhol): --capabilities does not allow passing multiple, we need to fix it # here we use samconfig-deep-nested.toml as a workaround config_file=self.test_data_path.joinpath("samconfig-deep-nested.toml"), - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, @@ -980,7 +981,7 @@ def test_deploy_create_failed_rollback(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, image_repository=self.ecr_repo_name, force_upload=True, @@ -1017,7 +1018,7 @@ def test_deploy_create_failed_disable_rollback(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, image_repository=self.ecr_repo_name, force_upload=True, @@ -1050,7 +1051,7 @@ def test_deploy_create_failed_disable_rollback(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, image_repository=self.ecr_repo_name, force_upload=True, @@ -1078,7 +1079,7 @@ def test_deploy_update_failed_rollback(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, image_repository=self.ecr_repo_name, force_upload=True, @@ -1099,7 +1100,7 @@ def test_deploy_update_failed_rollback(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, image_repository=self.ecr_repo_name, force_upload=True, @@ -1137,7 +1138,7 @@ def test_deploy_update_failed_disable_rollback(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, image_repository=self.ecr_repo_name, force_upload=True, @@ -1158,7 +1159,7 @@ def test_deploy_update_failed_disable_rollback(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, image_repository=self.ecr_repo_name, force_upload=True, @@ -1191,7 +1192,7 @@ def test_deploy_update_failed_disable_rollback(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, image_repository=self.ecr_repo_name, force_upload=True, @@ -1219,7 +1220,7 @@ def test_deploy_logs_warning_with_cdk_project(self, template_file): template_file=template_path, stack_name=stack_name, capabilities="CAPABILITY_IAM", - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, diff --git a/tests/integration/package/package_integ_base.py b/tests/integration/package/package_integ_base.py index 72301ed95b..957381986d 100644 --- a/tests/integration/package/package_integ_base.py +++ b/tests/integration/package/package_integ_base.py @@ -65,6 +65,7 @@ def setUpClass(cls): cls.kms_key = os.environ.get("AWS_KMS_KEY") # Use a pre-created S3 Bucket if present else create a new one cls.s3_bucket = s3.Bucket(cls.bucket_name) + cls.s3_prefix = uuid.uuid4().hex if not cls.pre_created_bucket: cls.s3_bucket.create() time.sleep(SLEEP) diff --git a/tests/integration/package/test_package_command_image.py b/tests/integration/package/test_package_command_image.py index 751f849bbe..287a4f28bb 100644 --- a/tests/integration/package/test_package_command_image.py +++ b/tests/integration/package/test_package_command_image.py @@ -176,7 +176,7 @@ def test_package_template_with_non_ecr_repo_uri_image_repository(self, template_ ) def test_package_template_and_s3_bucket(self, template_file): template_path = self.test_data_path.joinpath(template_file) - command_list = self.get_command_list(s3_bucket=self.s3_bucket, template=template_path) + command_list = self.get_command_list(s3_bucket=self.s3_bucket, s3_prefix=self.s3_prefix, template=template_path) process = Popen(command_list, stdout=PIPE, stderr=PIPE) try: diff --git a/tests/integration/package/test_package_command_zip.py b/tests/integration/package/test_package_command_zip.py index 05a0dc0ca2..4f3ab657d2 100644 --- a/tests/integration/package/test_package_command_zip.py +++ b/tests/integration/package/test_package_command_zip.py @@ -28,7 +28,9 @@ def tearDown(self): @parameterized.expand(["aws-serverless-function.yaml", "cdk_v1_synthesized_template_zip_functions.json"]) def test_package_template_flag(self, template_file): template_path = self.test_data_path.joinpath(template_file) - command_list = self.get_command_list(s3_bucket=self.s3_bucket.name, template=template_path) + command_list = self.get_command_list( + s3_bucket=self.s3_bucket.name, s3_prefix=self.s3_prefix, template=template_path + ) process = Popen(command_list, stdout=PIPE) try: @@ -48,7 +50,9 @@ def test_package_template_flag(self, template_file): ) def test_package_nested_template(self, template_file, uploading_count): template_path = self.test_data_path.joinpath(template_file) - command_list = self.get_command_list(s3_bucket=self.s3_bucket.name, template=template_path, force_upload=True) + command_list = self.get_command_list( + s3_bucket=self.s3_bucket.name, s3_prefix=self.s3_prefix, template=template_path, force_upload=True + ) process = Popen(command_list, stderr=PIPE) try: @@ -88,7 +92,9 @@ def test_package_nested_template(self, template_file, uploading_count): ) def test_package_barebones(self, template_file): template_path = self.test_data_path.joinpath(template_file) - command_list = self.get_command_list(s3_bucket=self.s3_bucket.name, template_file=template_path) + command_list = self.get_command_list( + s3_bucket=self.s3_bucket.name, s3_prefix=self.s3_prefix, template_file=template_path + ) process = Popen(command_list, stdout=PIPE) try: @@ -138,9 +144,8 @@ def test_package_without_required_args(self): ) def test_package_with_prefix(self, template_file): template_path = self.test_data_path.joinpath(template_file) - s3_prefix = "integ_test_prefix" command_list = self.get_command_list( - s3_bucket=self.s3_bucket.name, template_file=template_path, s3_prefix=s3_prefix + s3_bucket=self.s3_bucket.name, template_file=template_path, s3_prefix=self.s3_prefix ) process = Popen(command_list, stdout=PIPE) @@ -153,7 +158,7 @@ def test_package_with_prefix(self, template_file): self.assertIn("{bucket_name}".format(bucket_name=self.s3_bucket.name), process_stdout.decode("utf-8")) - self.assertIn("{s3_prefix}".format(s3_prefix=s3_prefix), process_stdout.decode("utf-8")) + self.assertIn("{s3_prefix}".format(s3_prefix=self.s3_prefix), process_stdout.decode("utf-8")) @parameterized.expand( [ @@ -182,14 +187,13 @@ def test_package_with_prefix(self, template_file): ) def test_package_with_output_template_file(self, template_file): template_path = self.test_data_path.joinpath(template_file) - s3_prefix = "integ_test_prefix" with tempfile.NamedTemporaryFile(delete=False) as output_template: command_list = self.get_command_list( s3_bucket=self.s3_bucket.name, template_file=template_path, - s3_prefix=s3_prefix, + s3_prefix=self.s3_prefix, output_template_file=output_template.name, ) @@ -238,14 +242,13 @@ def test_package_with_output_template_file(self, template_file): ) def test_package_with_json(self, template_file): template_path = self.test_data_path.joinpath(template_file) - s3_prefix = "integ_test_prefix" with tempfile.NamedTemporaryFile(delete=False) as output_template: command_list = self.get_command_list( s3_bucket=self.s3_bucket.name, template_file=template_path, - s3_prefix=s3_prefix, + s3_prefix=self.s3_prefix, output_template_file=output_template.name, use_json=True, ) @@ -295,7 +298,6 @@ def test_package_with_json(self, template_file): ) def test_package_with_force_upload(self, template_file): template_path = self.test_data_path.joinpath(template_file) - s3_prefix = "integ_test_prefix" with tempfile.NamedTemporaryFile(delete=False) as output_template: # Upload twice and see the string to have packaged artifacts both times. @@ -304,7 +306,7 @@ def test_package_with_force_upload(self, template_file): command_list = self.get_command_list( s3_bucket=self.s3_bucket.name, template_file=template_path, - s3_prefix=s3_prefix, + s3_prefix=self.s3_prefix, output_template_file=output_template.name, force_upload=True, ) @@ -354,13 +356,12 @@ def test_package_with_force_upload(self, template_file): ) def test_package_with_kms_key(self, template_file): template_path = self.test_data_path.joinpath(template_file) - s3_prefix = "integ_test_prefix" with tempfile.NamedTemporaryFile(delete=False) as output_template: command_list = self.get_command_list( s3_bucket=self.s3_bucket.name, template_file=template_path, - s3_prefix=s3_prefix, + s3_prefix=self.s3_prefix, output_template_file=output_template.name, force_upload=True, kms_key_id=self.kms_key, @@ -411,13 +412,12 @@ def test_package_with_kms_key(self, template_file): ) def test_package_with_metadata(self, template_file): template_path = self.test_data_path.joinpath(template_file) - s3_prefix = "integ_test_prefix" with tempfile.NamedTemporaryFile(delete=False) as output_template: command_list = self.get_command_list( s3_bucket=self.s3_bucket.name, template_file=template_path, - s3_prefix=s3_prefix, + s3_prefix=self.s3_prefix, output_template_file=output_template.name, force_upload=True, metadata={"integ": "yes"}, @@ -467,12 +467,11 @@ def test_package_with_metadata(self, template_file): ) def test_package_with_resolve_s3(self, template_file): template_path = self.test_data_path.joinpath(template_file) - s3_prefix = "integ_test_prefix" with tempfile.NamedTemporaryFile(delete=False) as output_template: command_list = self.get_command_list( template_file=template_path, - s3_prefix=s3_prefix, + s3_prefix=self.s3_prefix, output_template_file=output_template.name, force_upload=True, resolve_s3=True, @@ -499,12 +498,11 @@ def test_package_with_resolve_s3(self, template_file): @parameterized.expand([(True,), (False,)]) def test_package_with_no_progressbar(self, no_progressbar): template_path = self.test_data_path.joinpath("aws-serverless-function.yaml") - s3_prefix = "integ_test_prefix" with tempfile.NamedTemporaryFile(delete=False) as output_template: command_list = self.get_command_list( template_file=template_path, - s3_prefix=s3_prefix, + s3_prefix=self.s3_prefix, output_template_file=output_template.name, force_upload=True, no_progressbar=no_progressbar, @@ -539,7 +537,9 @@ def test_package_with_no_progressbar(self, no_progressbar): ) def test_package_with_warning_template(self, template_file, warning_keyword): template_path = self.test_data_path.joinpath(template_file) - command_list = self.get_command_list(s3_bucket=self.s3_bucket.name, template=template_path) + command_list = self.get_command_list( + s3_bucket=self.s3_bucket.name, s3_prefix=self.s3_prefix, template=template_path + ) process = Popen(command_list, stdout=PIPE) try: @@ -567,7 +567,9 @@ def test_package_with_deep_nested_template(self): template_file = os.path.join("deep-nested", "template.yaml") template_path = self.test_data_path.joinpath(template_file) - command_list = self.get_command_list(s3_bucket=self.s3_bucket.name, template=template_path, force_upload=True) + command_list = self.get_command_list( + s3_bucket=self.s3_bucket.name, s3_prefix=self.s3_prefix, template=template_path, force_upload=True + ) process = Popen(command_list, stdout=PIPE, stderr=PIPE) try: @@ -602,7 +604,9 @@ def test_package_with_deep_nested_template(self): @parameterized.expand(["aws-serverless-function-cdk.yaml", "cdk_v1_synthesized_template_zip_functions.json"]) def test_package_logs_warning_for_cdk_project(self, template_file): template_path = self.test_data_path.joinpath(template_file) - command_list = self.get_command_list(s3_bucket=self.s3_bucket.name, template_file=template_path) + command_list = self.get_command_list( + s3_bucket=self.s3_bucket.name, s3_prefix=self.s3_prefix, template_file=template_path + ) process = Popen(command_list, stdout=PIPE) try: diff --git a/tests/integration/sync/test_sync_command.py b/tests/integration/sync/test_sync_command.py index d9a6448d65..0ada30d9fa 100644 --- a/tests/integration/sync/test_sync_command.py +++ b/tests/integration/sync/test_sync_command.py @@ -100,7 +100,7 @@ def _test_sync_infra(self, runtime): stack_name=stack_name, parameter_overrides="Parameter=Clarity", image_repository=self.ecr_repo_name, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, kms_key_id=self.kms_key, tags="integ=true clarity=yes foo_bar=baz", ) @@ -137,7 +137,7 @@ def _test_sync_infra(self, runtime): stack_name=stack_name, parameter_overrides="Parameter=Clarity", image_repository=self.ecr_repo_name, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, kms_key_id=self.kms_key, tags="integ=true clarity=yes foo_bar=baz", ) @@ -176,7 +176,7 @@ def test_sync_infra_no_confirm(self, template_file): stack_name=stack_name, parameter_overrides="Parameter=Clarity", image_repository=self.ecr_repo_name, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, kms_key_id=self.kms_key, tags="integ=true clarity=yes foo_bar=baz", ) @@ -197,7 +197,7 @@ def test_sync_infra_no_stack_name(self, template_file): dependency_layer=True, parameter_overrides="Parameter=Clarity", image_repository=self.ecr_repo_name, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, kms_key_id=self.kms_key, tags="integ=true clarity=yes foo_bar=baz", ) @@ -221,7 +221,7 @@ def test_sync_infra_no_capabilities(self, template_file): stack_name=stack_name, parameter_overrides="Parameter=Clarity", image_repository=self.ecr_repo_name, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, kms_key_id=self.kms_key, capabilities="CAPABILITY_IAM", tags="integ=true clarity=yes foo_bar=baz", @@ -307,7 +307,7 @@ def test_cdk_templates(self, template_file, template_after, function_id, depende stack_name=stack_name, parameter_overrides="Parameter=Clarity", image_repositories=repository, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, kms_key_id=self.kms_key, tags="integ=true clarity=yes foo_bar=baz", ) @@ -335,7 +335,7 @@ def test_cdk_templates(self, template_file, template_after, function_id, depende stack_name=stack_name, parameter_overrides="Parameter=Clarity", image_repositories=repository, - s3_prefix="integ_deploy", + s3_prefix=self.s3_prefix, kms_key_id=self.kms_key, tags="integ=true clarity=yes foo_bar=baz", ) From 41fa022ff328caa1a0da842a8af5392d5a0361cd Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 5 Jan 2022 11:49:33 -0800 Subject: [PATCH 07/10] fix ruby version --- appveyor-ubuntu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor-ubuntu.yml b/appveyor-ubuntu.yml index 277293bdec..462731c568 100644 --- a/appveyor-ubuntu.yml +++ b/appveyor-ubuntu.yml @@ -74,7 +74,7 @@ install: - sh: "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" - sh: "PATH=$JAVA_HOME/bin:$PATH" - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate" - - sh: "rvm use 2.7" + - sh: "rvm use 2.7.2" - sh: "docker --version" # Install latest gradle From a056e523b91b106824b7f99afd0c223a5db9890f Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 5 Jan 2022 12:20:33 -0800 Subject: [PATCH 08/10] move s3 prefix creation to Setup method --- tests/integration/package/package_integ_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/package/package_integ_base.py b/tests/integration/package/package_integ_base.py index 957381986d..00fad65df6 100644 --- a/tests/integration/package/package_integ_base.py +++ b/tests/integration/package/package_integ_base.py @@ -65,7 +65,6 @@ def setUpClass(cls): cls.kms_key = os.environ.get("AWS_KMS_KEY") # Use a pre-created S3 Bucket if present else create a new one cls.s3_bucket = s3.Bucket(cls.bucket_name) - cls.s3_prefix = uuid.uuid4().hex if not cls.pre_created_bucket: cls.s3_bucket.create() time.sleep(SLEEP) @@ -78,6 +77,7 @@ def setUpClass(cls): time.sleep(SLEEP) def setUp(self): + self.s3_prefix = uuid.uuid4().hex super().setUp() def tearDown(self): From a338eb1c7830d8c81a13e55cca4f525f2782de78 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 5 Jan 2022 15:55:08 -0800 Subject: [PATCH 09/10] fix the sync integration testing --- appveyor-ubuntu-sync.yml | 126 ++++++++++++++++++++ appveyor-windows-sync.yml | 93 +++++++++++++++ tests/integration/sync/test_sync_command.py | 2 +- 3 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 appveyor-ubuntu-sync.yml create mode 100644 appveyor-windows-sync.yml diff --git a/appveyor-ubuntu-sync.yml b/appveyor-ubuntu-sync.yml new file mode 100644 index 0000000000..ed7702592e --- /dev/null +++ b/appveyor-ubuntu-sync.yml @@ -0,0 +1,126 @@ +version: 1.0.{build} +image: + - Ubuntu + +environment: + AWS_DEFAULT_REGION: us-east-1 + SAM_CLI_DEV: 1 + + matrix: + + - PYTHON_HOME: "C:\\Python36-x64" + PYTHON_VERSION: '3.6' + PYTHON_ARCH: '64' + NOSE_PARAMETERIZED_NO_WARN: 1 + INSTALL_PY_37_PIP: 1 + INSTALL_PY_38_PIP: 1 + INSTALL_PY_39_PIP: 1 + AWS_S3: 'AWS_S3_36' + AWS_ECR: 'AWS_ECR_36' + APPVEYOR_CONSOLE_DISABLE_PTY: true + + - PYTHON_HOME: "C:\\Python37-x64" + PYTHON_VERSION: '3.7' + PYTHON_ARCH: '64' + RUN_SMOKE: 1 + NOSE_PARAMETERIZED_NO_WARN: 1 + INSTALL_PY_36_PIP: 1 + INSTALL_PY_38_PIP: 1 + INSTALL_PY_39_PIP: 1 + AWS_S3: 'AWS_S3_37' + AWS_ECR: 'AWS_ECR_37' + APPVEYOR_CONSOLE_DISABLE_PTY: true + + - PYTHON_HOME: "C:\\Python38-x64" + PYTHON_VERSION: '3.8' + PYTHON_ARCH: '64' + RUN_SMOKE: 1 + NOSE_PARAMETERIZED_NO_WARN: 1 + INSTALL_PY_36_PIP: 1 + INSTALL_PY_37_PIP: 1 + INSTALL_PY_39_PIP: 1 + AWS_S3: 'AWS_S3_38' + AWS_ECR: 'AWS_ECR_38' + APPVEYOR_CONSOLE_DISABLE_PTY: true + + - PYTHON_HOME: "C:\\Python39-x64" + PYTHON_VERSION: '3.9' + PYTHON_ARCH: '64' + RUN_SMOKE: 1 + NOSE_PARAMETERIZED_NO_WARN: 1 + INSTALL_PY_36_PIP: 1 + INSTALL_PY_37_PIP: 1 + INSTALL_PY_38_PIP: 1 + AWS_S3: 'AWS_S3_39' + AWS_ECR: 'AWS_ECR_39' + APPVEYOR_CONSOLE_DISABLE_PTY: true + +install: + # apt repo for python3.9 installation + - sh: "sudo add-apt-repository ppa:deadsnakes/ppa" + # AppVeyor's apt-get cache might be outdated, and the package could potentially be 404. + - sh: "sudo apt-get update" + + - sh: "gvm use go1.13" + - sh: "echo $PATH" + - sh: "ls /usr/" + - sh: "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" + - sh: "PATH=$JAVA_HOME/bin:$PATH" + - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate" + - sh: "rvm use 2.7.2" + - sh: "docker --version" + + # Install latest gradle + - sh: "sudo apt-get -y remove gradle" + - sh: "wget https://services.gradle.org/distributions/gradle-5.5-bin.zip -P /tmp" + - sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip" + - sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH" + + # Install AWS CLI + - sh: "virtualenv aws_cli" + - sh: "./aws_cli/bin/python -m pip install awscli" + - sh: "PATH=$(echo $PWD'/aws_cli/bin'):$PATH" + + - sh: "sudo apt-get -y install python3.6" + - sh: "sudo apt-get -y install python2.7" + - sh: "sudo apt-get -y install python3.7" + - sh: "sudo apt-get -y install python3.8" + - sh: "sudo apt-get -y install python3.9" + + - sh: "which python3.8" + - sh: "which python3.7" + - sh: "which python3.6" + - sh: "which python3.9" + - sh: "which python2.7" + + - sh: "PATH=$PATH:/usr/bin/python3.9:/usr/bin/python3.8:/usr/bin/python3.7" + - sh: "curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py" + + - sh: "sudo apt-get -y install python3-distutils" + - sh: "sudo apt-get -y install python3.9-distutils" + - ps: "If ($env:INSTALL_PY_39_PIP) {python3.9 get-pip.py --user}" + - ps: "If ($env:INSTALL_PY_38_PIP) {python3.8 get-pip.py --user}" + - ps: "If ($env:INSTALL_PY_37_PIP) {python3.7 get-pip.py --user}" + - ps: "If ($env:INSTALL_PY_36_PIP) {python3.6 get-pip.py --user}" + + # required for RIE with arm64 in linux + - sh: "docker run --rm --privileged multiarch/qemu-user-static --reset -p yes" + + # update ca-certificates which causes failures with newest golang library + - sh: "sudo apt-get install --reinstall ca-certificates" + +build_script: + - "python -c \"import sys; print(sys.executable)\"" + +test_script: + # Pre-dev Tests + - "pip install -e \".[pre-dev]\"" + + # Runs only in Linux, logging Public ECR when running canary and cred is available + - sh: " + if [[ -n $BY_CANARY ]]; + then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; + fi" + + - "pip install -e \".[dev]\"" + - sh: "pytest -vv tests/integration/sync" diff --git a/appveyor-windows-sync.yml b/appveyor-windows-sync.yml new file mode 100644 index 0000000000..4e8ec323e1 --- /dev/null +++ b/appveyor-windows-sync.yml @@ -0,0 +1,93 @@ +version: 1.0.{build} +image: Windows Server 2019 +build: off + +# Change the clone folder to somewhere in "D:\" because this is shared by default with Docker. We need this to mount folders. +clone_folder: D:\source + +environment: + AWS_DEFAULT_REGION: us-east-1 + SAM_CLI_DEV: 1 + + # In Windows, tempdir is usually in C:\. But in AppVeyor only D:\ is shared in Docker. + # Therefore change TEMPDIR it to D: for invoke tests to work. Python uses $TMPDIR envvar + # to find root of tempdir + TMPDIR: D:\tmp + TEMP: D:\tmp + TMP: D:\tmp + + # MSI Installers only use Py3.7.6. It is sufficient to test with this version here. + PYTHON_HOME: "C:\\Python37-x64" + PYTHON_SCRIPTS: "C:\\Python37-x64\\Scripts" + PYTHON_EXE: "C:\\Python37-x64\\python.exe" + PYTHON_ARCH: '64' + HOME: 'C:\Users\appveyor' + HOMEDRIVE: 'C:' + HOMEPATH: 'C:\Users\appveyor' + NOSE_PARAMETERIZED_NO_WARN: 1 + AWS_S3: 'AWS_S3_37_WIN' + AWS_ECR: 'AWS_ECR_37' + APPVEYOR_CONSOLE_DISABLE_PTY: true + +init: + # Uncomment this for RDP + - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + - ps: gcim Win32_Processor | % { "$($_.NumberOfLogicalProcessors) logical CPUs" } + - ps: gcim Win32_OperatingSystem | % { "$([int]($_.TotalVisibleMemorySize/1mb)) Gb" } + +cache: + - C:\ProgramData\chocolatey\bin -> appveyor.yml + - C:\ProgramData\chocolatey\lib -> appveyor.yml + +install: + # setup make + - "choco install make" + + # Make sure the temp directory exists for Python to use. + - ps: "mkdir -Force D:\\tmp" + - "SET PATH=%PYTHON_HOME%;%PATH%" + - "echo %PYTHON_HOME%" + - "echo %PATH%" + - "python --version" + - "docker info" + + # Upgrade setuptools, wheel and virtualenv + - "python -m pip install --upgrade setuptools wheel virtualenv" + + # Install AWS CLI Globally via pip3 + - "pip install awscli" + + # Create new virtual environment with chosen python version and activate it + - "python -m virtualenv venv" + - "venv\\Scripts\\activate" + - "python --version" + + # Actually install SAM CLI's dependencies + - "pip install -e \".[dev]\"" + + # Install aws cli + - "pip install awscli" + + # Switch to Docker Linux containers + - ps: Switch-DockerLinux + + # Check for git executable + - "git --version" + + # Echo final Path + - "echo %PATH%" + +test_script: + # Reactivate virtualenv before running tests + - ps: " + If (Test-Path env:BY_CANARY){ + echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; + }" + - "git --version" + - "venv\\Scripts\\activate" + - "docker system prune -a -f" + - "pytest -vv tests/integration/sync" + +# Uncomment for RDP +# on_finish: +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) diff --git a/tests/integration/sync/test_sync_command.py b/tests/integration/sync/test_sync_command.py index 0ada30d9fa..54d4f6e2c1 100644 --- a/tests/integration/sync/test_sync_command.py +++ b/tests/integration/sync/test_sync_command.py @@ -40,7 +40,7 @@ @skipIf(SKIP_SYNC_TESTS, "Skip sync tests in CI/CD only") -class TestSync(BuildIntegBase, SyncIntegBase, PackageIntegBase): +class TestSync(PackageIntegBase, BuildIntegBase, SyncIntegBase): @classmethod def setUpClass(cls): PackageIntegBase.setUpClass() From c2d9567d486290c3ac89c9002e2b35ac9f3350d8 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 5 Jan 2022 16:47:34 -0800 Subject: [PATCH 10/10] remove appveyor sync --- appveyor-ubuntu-sync.yml | 126 -------------------------------------- appveyor-windows-sync.yml | 93 ---------------------------- 2 files changed, 219 deletions(-) delete mode 100644 appveyor-ubuntu-sync.yml delete mode 100644 appveyor-windows-sync.yml diff --git a/appveyor-ubuntu-sync.yml b/appveyor-ubuntu-sync.yml deleted file mode 100644 index ed7702592e..0000000000 --- a/appveyor-ubuntu-sync.yml +++ /dev/null @@ -1,126 +0,0 @@ -version: 1.0.{build} -image: - - Ubuntu - -environment: - AWS_DEFAULT_REGION: us-east-1 - SAM_CLI_DEV: 1 - - matrix: - - - PYTHON_HOME: "C:\\Python36-x64" - PYTHON_VERSION: '3.6' - PYTHON_ARCH: '64' - NOSE_PARAMETERIZED_NO_WARN: 1 - INSTALL_PY_37_PIP: 1 - INSTALL_PY_38_PIP: 1 - INSTALL_PY_39_PIP: 1 - AWS_S3: 'AWS_S3_36' - AWS_ECR: 'AWS_ECR_36' - APPVEYOR_CONSOLE_DISABLE_PTY: true - - - PYTHON_HOME: "C:\\Python37-x64" - PYTHON_VERSION: '3.7' - PYTHON_ARCH: '64' - RUN_SMOKE: 1 - NOSE_PARAMETERIZED_NO_WARN: 1 - INSTALL_PY_36_PIP: 1 - INSTALL_PY_38_PIP: 1 - INSTALL_PY_39_PIP: 1 - AWS_S3: 'AWS_S3_37' - AWS_ECR: 'AWS_ECR_37' - APPVEYOR_CONSOLE_DISABLE_PTY: true - - - PYTHON_HOME: "C:\\Python38-x64" - PYTHON_VERSION: '3.8' - PYTHON_ARCH: '64' - RUN_SMOKE: 1 - NOSE_PARAMETERIZED_NO_WARN: 1 - INSTALL_PY_36_PIP: 1 - INSTALL_PY_37_PIP: 1 - INSTALL_PY_39_PIP: 1 - AWS_S3: 'AWS_S3_38' - AWS_ECR: 'AWS_ECR_38' - APPVEYOR_CONSOLE_DISABLE_PTY: true - - - PYTHON_HOME: "C:\\Python39-x64" - PYTHON_VERSION: '3.9' - PYTHON_ARCH: '64' - RUN_SMOKE: 1 - NOSE_PARAMETERIZED_NO_WARN: 1 - INSTALL_PY_36_PIP: 1 - INSTALL_PY_37_PIP: 1 - INSTALL_PY_38_PIP: 1 - AWS_S3: 'AWS_S3_39' - AWS_ECR: 'AWS_ECR_39' - APPVEYOR_CONSOLE_DISABLE_PTY: true - -install: - # apt repo for python3.9 installation - - sh: "sudo add-apt-repository ppa:deadsnakes/ppa" - # AppVeyor's apt-get cache might be outdated, and the package could potentially be 404. - - sh: "sudo apt-get update" - - - sh: "gvm use go1.13" - - sh: "echo $PATH" - - sh: "ls /usr/" - - sh: "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" - - sh: "PATH=$JAVA_HOME/bin:$PATH" - - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate" - - sh: "rvm use 2.7.2" - - sh: "docker --version" - - # Install latest gradle - - sh: "sudo apt-get -y remove gradle" - - sh: "wget https://services.gradle.org/distributions/gradle-5.5-bin.zip -P /tmp" - - sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip" - - sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH" - - # Install AWS CLI - - sh: "virtualenv aws_cli" - - sh: "./aws_cli/bin/python -m pip install awscli" - - sh: "PATH=$(echo $PWD'/aws_cli/bin'):$PATH" - - - sh: "sudo apt-get -y install python3.6" - - sh: "sudo apt-get -y install python2.7" - - sh: "sudo apt-get -y install python3.7" - - sh: "sudo apt-get -y install python3.8" - - sh: "sudo apt-get -y install python3.9" - - - sh: "which python3.8" - - sh: "which python3.7" - - sh: "which python3.6" - - sh: "which python3.9" - - sh: "which python2.7" - - - sh: "PATH=$PATH:/usr/bin/python3.9:/usr/bin/python3.8:/usr/bin/python3.7" - - sh: "curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py" - - - sh: "sudo apt-get -y install python3-distutils" - - sh: "sudo apt-get -y install python3.9-distutils" - - ps: "If ($env:INSTALL_PY_39_PIP) {python3.9 get-pip.py --user}" - - ps: "If ($env:INSTALL_PY_38_PIP) {python3.8 get-pip.py --user}" - - ps: "If ($env:INSTALL_PY_37_PIP) {python3.7 get-pip.py --user}" - - ps: "If ($env:INSTALL_PY_36_PIP) {python3.6 get-pip.py --user}" - - # required for RIE with arm64 in linux - - sh: "docker run --rm --privileged multiarch/qemu-user-static --reset -p yes" - - # update ca-certificates which causes failures with newest golang library - - sh: "sudo apt-get install --reinstall ca-certificates" - -build_script: - - "python -c \"import sys; print(sys.executable)\"" - -test_script: - # Pre-dev Tests - - "pip install -e \".[pre-dev]\"" - - # Runs only in Linux, logging Public ECR when running canary and cred is available - - sh: " - if [[ -n $BY_CANARY ]]; - then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; - fi" - - - "pip install -e \".[dev]\"" - - sh: "pytest -vv tests/integration/sync" diff --git a/appveyor-windows-sync.yml b/appveyor-windows-sync.yml deleted file mode 100644 index 4e8ec323e1..0000000000 --- a/appveyor-windows-sync.yml +++ /dev/null @@ -1,93 +0,0 @@ -version: 1.0.{build} -image: Windows Server 2019 -build: off - -# Change the clone folder to somewhere in "D:\" because this is shared by default with Docker. We need this to mount folders. -clone_folder: D:\source - -environment: - AWS_DEFAULT_REGION: us-east-1 - SAM_CLI_DEV: 1 - - # In Windows, tempdir is usually in C:\. But in AppVeyor only D:\ is shared in Docker. - # Therefore change TEMPDIR it to D: for invoke tests to work. Python uses $TMPDIR envvar - # to find root of tempdir - TMPDIR: D:\tmp - TEMP: D:\tmp - TMP: D:\tmp - - # MSI Installers only use Py3.7.6. It is sufficient to test with this version here. - PYTHON_HOME: "C:\\Python37-x64" - PYTHON_SCRIPTS: "C:\\Python37-x64\\Scripts" - PYTHON_EXE: "C:\\Python37-x64\\python.exe" - PYTHON_ARCH: '64' - HOME: 'C:\Users\appveyor' - HOMEDRIVE: 'C:' - HOMEPATH: 'C:\Users\appveyor' - NOSE_PARAMETERIZED_NO_WARN: 1 - AWS_S3: 'AWS_S3_37_WIN' - AWS_ECR: 'AWS_ECR_37' - APPVEYOR_CONSOLE_DISABLE_PTY: true - -init: - # Uncomment this for RDP - - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - - ps: gcim Win32_Processor | % { "$($_.NumberOfLogicalProcessors) logical CPUs" } - - ps: gcim Win32_OperatingSystem | % { "$([int]($_.TotalVisibleMemorySize/1mb)) Gb" } - -cache: - - C:\ProgramData\chocolatey\bin -> appveyor.yml - - C:\ProgramData\chocolatey\lib -> appveyor.yml - -install: - # setup make - - "choco install make" - - # Make sure the temp directory exists for Python to use. - - ps: "mkdir -Force D:\\tmp" - - "SET PATH=%PYTHON_HOME%;%PATH%" - - "echo %PYTHON_HOME%" - - "echo %PATH%" - - "python --version" - - "docker info" - - # Upgrade setuptools, wheel and virtualenv - - "python -m pip install --upgrade setuptools wheel virtualenv" - - # Install AWS CLI Globally via pip3 - - "pip install awscli" - - # Create new virtual environment with chosen python version and activate it - - "python -m virtualenv venv" - - "venv\\Scripts\\activate" - - "python --version" - - # Actually install SAM CLI's dependencies - - "pip install -e \".[dev]\"" - - # Install aws cli - - "pip install awscli" - - # Switch to Docker Linux containers - - ps: Switch-DockerLinux - - # Check for git executable - - "git --version" - - # Echo final Path - - "echo %PATH%" - -test_script: - # Reactivate virtualenv before running tests - - ps: " - If (Test-Path env:BY_CANARY){ - echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; - }" - - "git --version" - - "venv\\Scripts\\activate" - - "docker system prune -a -f" - - "pytest -vv tests/integration/sync" - -# Uncomment for RDP -# on_finish: -# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))