diff --git a/samcli/commands/sync/command.py b/samcli/commands/sync/command.py index c5cbf8962a..fac24d7838 100644 --- a/samcli/commands/sync/command.py +++ b/samcli/commands/sync/command.py @@ -63,7 +63,15 @@ By default, the sync command runs a full stack update. You can specify --code or --watch to switch modes. \b Sync also supports nested stacks and nested stack resources. For example -$ sam sync --code --stack-name {stack} --resource-id {ChildStack}/{ResourceId} + +$ sam sync --code --stack-name {stack} --resource-id \\ +{ChildStack}/{ResourceId} + +Running --watch with --code option will provide a way to run code synchronization only, that will speed up start time +and will skip any template change. Please remember to update your deployed stack by running without --code option. + +$ sam sync --code --watch --stack-name {stack} + """ SYNC_INFO_TEXT = """ @@ -93,14 +101,12 @@ is_flag=True, help="Sync code resources. This includes Lambda Functions, API Gateway, and Step Functions.", cls=ClickMutex, - incompatible_params=["watch"], ) @click.option( "--watch", is_flag=True, help="Watch local files and automatically sync with remote.", cls=ClickMutex, - incompatible_params=["code"], ) @click.option( "--resource-id", @@ -340,7 +346,7 @@ def do_cli( with SyncContext(dependency_layer, build_context.build_dir, build_context.cache_dir): if watch: execute_watch( - template_file, build_context, package_context, deploy_context, dependency_layer + template_file, build_context, package_context, deploy_context, dependency_layer, code ) elif code: execute_code_sync( @@ -425,6 +431,7 @@ def execute_watch( package_context: "PackageContext", deploy_context: "DeployContext", auto_dependency_layer: bool, + skip_infra_syncs: bool, ): """Start sync watch execution @@ -439,7 +446,9 @@ def execute_watch( deploy_context : DeployContext DeployContext """ - watch_manager = WatchManager(template, build_context, package_context, deploy_context, auto_dependency_layer) + watch_manager = WatchManager( + template, build_context, package_context, deploy_context, auto_dependency_layer, skip_infra_syncs + ) watch_manager.start() diff --git a/samcli/lib/sync/sync_flow_factory.py b/samcli/lib/sync/sync_flow_factory.py index 8f0ffe617c..992c7da4c0 100644 --- a/samcli/lib/sync/sync_flow_factory.py +++ b/samcli/lib/sync/sync_flow_factory.py @@ -2,6 +2,9 @@ import logging from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, cast +from botocore.exceptions import ClientError + +from samcli.commands.exceptions import InvalidStackNameException from samcli.lib.bootstrap.nested_stack.nested_stack_manager import NestedStackManager from samcli.lib.providers.provider import Stack, get_resource_by_id, ResourceIdentifier from samcli.lib.sync.flows.auto_dependency_layer_sync_flow import AutoDependencyLayerParentSyncFlow @@ -16,7 +19,11 @@ from samcli.lib.sync.flows.rest_api_sync_flow import RestApiSyncFlow from samcli.lib.sync.flows.http_api_sync_flow import HttpApiSyncFlow from samcli.lib.sync.flows.stepfunctions_sync_flow import StepFunctionsSyncFlow -from samcli.lib.utils.boto_utils import get_boto_resource_provider_with_config, get_boto_client_provider_with_config +from samcli.lib.utils.boto_utils import ( + get_boto_resource_provider_with_config, + get_boto_client_provider_with_config, + get_client_error_code, +) from samcli.lib.utils.cloudformation import get_resource_summaries from samcli.lib.utils.resources import ( AWS_SERVERLESS_FUNCTION, @@ -110,11 +117,19 @@ def load_physical_id_mapping(self) -> None: region=self._deploy_context.region, profile=self._deploy_context.profile ) - resource_mapping = get_resource_summaries( - boto_resource_provider=resource_provider, - boto_client_provider=client_provider, - stack_name=self._deploy_context.stack_name, - ) + try: + resource_mapping = get_resource_summaries( + boto_resource_provider=resource_provider, + boto_client_provider=client_provider, + stack_name=self._deploy_context.stack_name, + ) + except ClientError as ex: + error_code = get_client_error_code(ex) + if error_code == "ValidationError": + raise InvalidStackNameException( + f"Invalid --stack-name parameter. Stack with id '{self._deploy_context.stack_name}' does not exist" + ) from ex + raise ex # get the resource_id -> physical_id mapping self._physical_id_mapping = { diff --git a/samcli/lib/sync/watch_manager.py b/samcli/lib/sync/watch_manager.py index 0171c4bbc2..ab4627721d 100644 --- a/samcli/lib/sync/watch_manager.py +++ b/samcli/lib/sync/watch_manager.py @@ -44,6 +44,7 @@ class WatchManager: _waiting_infra_sync: bool _color: Colored _auto_dependency_layer: bool + _skip_infra_syncs: bool def __init__( self, @@ -52,6 +53,7 @@ def __init__( package_context: "PackageContext", deploy_context: "DeployContext", auto_dependency_layer: bool, + skip_infra_syncs: bool, ): """Manager for sync watch execution logic. This manager will observe template and its code resources. @@ -74,6 +76,7 @@ def __init__( self._package_context = package_context self._deploy_context = deploy_context self._auto_dependency_layer = auto_dependency_layer + self._skip_infra_syncs = skip_infra_syncs self._sync_flow_factory = None self._sync_flow_executor = ContinuousSyncFlowExecutor() @@ -89,6 +92,14 @@ def queue_infra_sync(self) -> None: """Queue up an infra structure sync. A simple bool flag is suffice """ + if self._skip_infra_syncs: + LOG.info( + self._color.yellow( + "You have enabled the --code flag, which limits sam sync updates to code changes only. To do a " + "complete infrastructure and code sync, remove the --code flag." + ) + ) + return self._waiting_infra_sync = True def _update_stacks(self) -> None: @@ -166,6 +177,9 @@ def start(self) -> None: # This is a wrapper for gracefully handling Ctrl+C or other termination cases. try: self.queue_infra_sync() + if self._skip_infra_syncs: + self._start_sync() + LOG.info(self._color.green("Sync watch started.")) self._start() except KeyboardInterrupt: LOG.info(self._color.cyan("Shutting down sync watch...")) @@ -181,6 +195,16 @@ def _start(self) -> None: self._execute_infra_sync() time.sleep(1) + def _start_sync(self): + """ + Update stacks and populate all triggers + """ + self._observer.unschedule_all() + self._update_stacks() + self._add_template_triggers() + self._add_code_triggers() + self._start_code_sync() + def _execute_infra_sync(self) -> None: LOG.info(self._color.cyan("Queued infra sync. Waiting for in progress code syncs to complete...")) self._waiting_infra_sync = False @@ -188,6 +212,7 @@ def _execute_infra_sync(self) -> None: try: LOG.info(self._color.cyan("Starting infra sync.")) self._execute_infra_context() + LOG.info(self._color.green("Infra sync completed.")) except Exception as e: LOG.error( self._color.red("Failed to sync infra. Code sync is paused until template/stack is fixed."), @@ -197,15 +222,9 @@ def _execute_infra_sync(self) -> None: self._observer.unschedule_all() self._add_template_triggers() else: - # Update stacks and repopulate triggers # Trigger are not removed until infra sync is finished as there # can be code changes during infra sync. - self._observer.unschedule_all() - self._update_stacks() - self._add_template_triggers() - self._add_code_triggers() - self._start_code_sync() - LOG.info(self._color.green("Infra sync completed.")) + self._start_sync() def _on_code_change_wrapper(self, resource_id: ResourceIdentifier) -> OnChangeCallback: """Wrapper method that generates a callback for code changes. diff --git a/tests/integration/sync/test_sync_watch.py b/tests/integration/sync/test_sync_watch.py index cc26de3ffb..2385ca6d36 100644 --- a/tests/integration/sync/test_sync_watch.py +++ b/tests/integration/sync/test_sync_watch.py @@ -468,3 +468,103 @@ def test_sync_watch_code(self): lambda_response = json.loads(self._get_lambda_response(lambda_function)) self.assertIn("extra_message", lambda_response) self.assertEqual(lambda_response.get("message"), "7") + + +@parameterized_class( + [{"runtime": "python", "dependency_layer": True}, {"runtime": "python", "dependency_layer": False}] +) +class TestSyncWatchCodeOnly(TestSyncWatchBase): + template_before = str(Path("code", "before", "template-python-code-only.yaml")) + + def run_initial_infra_validation(self) -> None: + """Runs initial infra validation after deployment is completed""" + self.stack_resources = self._get_stacks(self.stack_name) + lambda_functions = self.stack_resources.get(AWS_LAMBDA_FUNCTION) + for lambda_function in lambda_functions: + lambda_response = json.loads(self._get_lambda_response(lambda_function)) + self.assertIn("extra_message", lambda_response) + self.assertEqual(lambda_response.get("message"), "7") + + def test_sync_watch_code(self): + # first kill previously started sync process + kill_process(self.watch_process) + # start new one with code only + template_path = self.test_dir.joinpath(self.template_before) + sync_command_list = self.get_sync_command_list( + template_file=str(template_path), + code=True, + watch=True, + dependency_layer=self.dependency_layer, + stack_name=self.stack_name, + parameter_overrides="Parameter=Clarity", + image_repository=self.ecr_repo_name, + s3_prefix=self.s3_prefix, + kms_key_id=self.kms_key, + tags="integ=true clarity=yes foo_bar=baz", + ) + self.watch_process = start_persistent_process(sync_command_list, cwd=self.test_dir) + read_until_string(self.watch_process, "Enter Y to proceed with the command, or enter N to cancel:\n") + + self.watch_process.stdin.write("y\n") + read_until_string(self.watch_process, "\x1b[32mSync watch started.\x1b[0m\n", timeout=30) + + self.stack_resources = self._get_stacks(self.stack_name) + + if self.dependency_layer: + # Test update manifest + layer_contents = self.get_dependency_layer_contents_from_arn(self.stack_resources, "python", 1) + self.assertNotIn("requests", layer_contents) + self.update_file( + self.test_dir.joinpath("code", "after", "function", "requirements.txt"), + self.test_dir.joinpath("code", "before", "function", "requirements.txt"), + ) + read_until_string( + self.watch_process, + "\x1b[32mFinished syncing Function Layer Reference Sync HelloWorldFunction.\x1b[0m\n", + timeout=45, + ) + layer_contents = self.get_dependency_layer_contents_from_arn(self.stack_resources, "python", 2) + self.assertIn("requests", layer_contents) + + # Test Lambda Function + self.update_file( + self.test_dir.joinpath("code", "after", "function", "app.py"), + self.test_dir.joinpath("code", "before", "function", "app.py"), + ) + read_until_string( + self.watch_process, "\x1b[32mFinished syncing Lambda Function HelloWorldFunction.\x1b[0m\n", timeout=30 + ) + lambda_functions = self.stack_resources.get(AWS_LAMBDA_FUNCTION) + for lambda_function in lambda_functions: + lambda_response = json.loads(self._get_lambda_response(lambda_function)) + self.assertIn("extra_message", lambda_response) + self.assertEqual(lambda_response.get("message"), "8") + + # Test Lambda Layer + self.update_file( + self.test_dir.joinpath("code", "after", "layer", "layer_method.py"), + self.test_dir.joinpath("code", "before", "layer", "layer_method.py"), + ) + read_until_string( + self.watch_process, + "\x1b[32mFinished syncing Function Layer Reference Sync HelloWorldFunction.\x1b[0m\n", + timeout=30, + ) + lambda_functions = self.stack_resources.get(AWS_LAMBDA_FUNCTION) + for lambda_function in lambda_functions: + lambda_response = json.loads(self._get_lambda_response(lambda_function)) + self.assertIn("extra_message", lambda_response) + self.assertEqual(lambda_response.get("message"), "9") + + # updating infra should not trigger an infra sync + self.update_file( + self.test_dir.joinpath(f"infra/template-{self.runtime}-after.yaml"), + self.test_dir.joinpath(f"code/before/template-{self.runtime}-code-only.yaml"), + ) + + read_until_string( + self.watch_process, + "\x1b[33mYou have enabled the --code flag, which limits sam sync updates to code changes only. To do a " + "complete infrastructure and code sync, remove the --code flag.\x1b[0m\n", + timeout=30, + ) diff --git a/tests/integration/testdata/sync/code/before/template-python-code-only.yaml b/tests/integration/testdata/sync/code/before/template-python-code-only.yaml new file mode 100644 index 0000000000..f788e2e386 --- /dev/null +++ b/tests/integration/testdata/sync/code/before/template-python-code-only.yaml @@ -0,0 +1,28 @@ +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 + +Globals: + Function: + Timeout: 10 + +Resources: + HelloWorldFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: function/ + Handler: app.lambda_handler + Runtime: python3.7 + Layers: + - Ref: HelloWorldLayer + Tracing: Active + + HelloWorldLayer: + Type: AWS::Serverless::LayerVersion + Properties: + LayerName: HelloWorldLayer + Description: Hello World Layer + ContentUri: layer/ + CompatibleRuntimes: + - python3.7 + Metadata: + BuildMethod: python3.7 diff --git a/tests/unit/commands/sync/test_command.py b/tests/unit/commands/sync/test_command.py index dbabc332dd..e9da1588d2 100644 --- a/tests/unit/commands/sync/test_command.py +++ b/tests/unit/commands/sync/test_command.py @@ -1,3 +1,4 @@ +import itertools import os from unittest import TestCase from unittest.mock import ANY, MagicMock, Mock, patch @@ -229,7 +230,7 @@ def test_watch_must_succeed_sync( execute_watch_mock, click_mock, ): - + skip_infra_syncs = watch and code build_context_mock = Mock() BuildContextMock.return_value.__enter__.return_value = build_context_mock package_context_mock = Mock() @@ -328,7 +329,12 @@ def test_watch_must_succeed_sync( on_failure=None, ) execute_watch_mock.assert_called_once_with( - self.template_file, build_context_mock, package_context_mock, deploy_context_mock, auto_dependency_layer + self.template_file, + build_context_mock, + package_context_mock, + deploy_context_mock, + auto_dependency_layer, + skip_infra_syncs, ) @parameterized.expand([(True, False, True, False), (True, False, False, True)]) @@ -650,21 +656,33 @@ def setUp(self) -> None: self.package_context = MagicMock() self.deploy_context = MagicMock() - @parameterized.expand([(True,), (False,)]) + @parameterized.expand(itertools.product([True, False], [True, False])) @patch("samcli.commands.sync.command.click") @patch("samcli.commands.sync.command.WatchManager") def test_execute_watch( self, + code, auto_dependency_layer, watch_manager_mock, click_mock, ): + skip_infra_syncs = code execute_watch( - self.template_file, self.build_context, self.package_context, self.deploy_context, auto_dependency_layer + self.template_file, + self.build_context, + self.package_context, + self.deploy_context, + auto_dependency_layer, + skip_infra_syncs, ) watch_manager_mock.assert_called_once_with( - self.template_file, self.build_context, self.package_context, self.deploy_context, auto_dependency_layer + self.template_file, + self.build_context, + self.package_context, + self.deploy_context, + auto_dependency_layer, + skip_infra_syncs, ) watch_manager_mock.return_value.start.assert_called_once_with() diff --git a/tests/unit/lib/sync/test_watch_manager.py b/tests/unit/lib/sync/test_watch_manager.py index f7e7a3acaf..7eaec90f65 100644 --- a/tests/unit/lib/sync/test_watch_manager.py +++ b/tests/unit/lib/sync/test_watch_manager.py @@ -21,7 +21,7 @@ def setUp(self) -> None: self.package_context = MagicMock() self.deploy_context = MagicMock() self.watch_manager = WatchManager( - self.template, self.build_context, self.package_context, self.deploy_context, False + self.template, self.build_context, self.package_context, self.deploy_context, False, False ) def tearDown(self) -> None: @@ -227,6 +227,48 @@ def test__start(self, sleep_mock): self.path_observer.start.assert_called_once_with() + @patch("samcli.lib.sync.watch_manager.time.sleep") + def test_start_code_only(self, sleep_mock): + sleep_mock.side_effect = KeyboardInterrupt() + + stop_code_sync_mock = MagicMock() + execute_infra_sync_mock = MagicMock() + + update_stacks_mock = MagicMock() + add_template_trigger_mock = MagicMock() + add_code_trigger_mock = MagicMock() + start_code_sync_mock = MagicMock() + + self.watch_manager._stop_code_sync = stop_code_sync_mock + self.watch_manager._execute_infra_context = execute_infra_sync_mock + self.watch_manager._update_stacks = update_stacks_mock + self.watch_manager._add_template_triggers = add_template_trigger_mock + self.watch_manager._add_code_triggers = add_code_trigger_mock + self.watch_manager._start_code_sync = start_code_sync_mock + + self.watch_manager._skip_infra_syncs = True + with self.assertRaises(KeyboardInterrupt): + self.watch_manager._start() + + self.path_observer.start.assert_called_once_with() + self.assertFalse(self.watch_manager._waiting_infra_sync) + + stop_code_sync_mock.assert_not_called() + execute_infra_sync_mock.assert_not_called() + update_stacks_mock.assert_not_called() + add_template_trigger_mock.assert_not_called() + add_code_trigger_mock.assert_not_called() + start_code_sync_mock.assert_not_called() + + self.path_observer.unschedule_all.assert_not_called() + + self.path_observer.start.assert_called_once_with() + + def test_start_code_only_infra_sync_not_set(self): + self.watch_manager._skip_infra_syncs = True + self.watch_manager.queue_infra_sync() + self.assertFalse(self.watch_manager._waiting_infra_sync) + @patch("samcli.lib.sync.watch_manager.time.sleep") def test__start_infra_exception(self, sleep_mock): sleep_mock.side_effect = KeyboardInterrupt()