From feff410d2cb3fe24dc2a4457c6b4e6c2befd8508 Mon Sep 17 00:00:00 2001 From: Jacob Fuss Date: Mon, 22 Apr 2019 20:26:33 -0700 Subject: [PATCH 1/2] feat(build): Build a single function --- designs/sam_build_cmd.md | 3 +- samcli/commands/build/build_context.py | 22 +++++ samcli/commands/build/command.py | 27 ++++--- samcli/lib/build/app_builder.py | 10 +-- .../integration/buildcmd/build_integ_base.py | 12 ++- tests/integration/buildcmd/test_build_cmd.py | 80 +++++++++++++++++++ .../buildcmd/many-functions-template.yaml | 28 +++++++ .../commands/buildcmd/test_build_context.py | 60 +++++++++++++- tests/unit/commands/buildcmd/test_command.py | 21 ++++- .../unit/lib/build_module/test_app_builder.py | 16 ++-- 10 files changed, 246 insertions(+), 33 deletions(-) create mode 100644 tests/integration/testdata/buildcmd/many-functions-template.yaml diff --git a/designs/sam_build_cmd.md b/designs/sam_build_cmd.md index b4cfb86ac5..6624977d07 100644 --- a/designs/sam_build_cmd.md +++ b/designs/sam_build_cmd.md @@ -50,7 +50,8 @@ Success criteria for the change - Python with PIP - Golang with Go CLI - Dotnetcore with DotNet CLI -2. Each Lambda function in SAM template gets built +2. Each Lambda function in SAM template gets built by default unless a `function_identifier` (LogicalID) is passed + to the build command 3. Produce stable builds (best effort): If the source files did not change, built artifacts should not change. 4. Built artifacts should \"just work\" with `sam local` and diff --git a/samcli/commands/build/build_context.py b/samcli/commands/build/build_context.py index dfed756294..0cb547f0ca 100644 --- a/samcli/commands/build/build_context.py +++ b/samcli/commands/build/build_context.py @@ -2,6 +2,7 @@ Context object used by build command """ +import logging import os import shutil @@ -14,6 +15,9 @@ from samcli.commands.local.lib.sam_function_provider import SamFunctionProvider from samcli.commands._utils.template import get_template_data from samcli.commands.exceptions import UserException +from samcli.local.lambdafn.exceptions import FunctionNotFound + +LOG = logging.getLogger(__name__) class BuildContext(object): @@ -23,6 +27,7 @@ class BuildContext(object): _BUILD_DIR_PERMISSIONS = 0o755 def __init__(self, + function_identifier, template_file, base_dir, build_dir, @@ -34,6 +39,7 @@ def __init__(self, docker_network=None, skip_pull_image=False): + self._function_identifier = function_identifier self._template_file = template_file self._base_dir = base_dir self._build_dir = build_dir @@ -128,3 +134,19 @@ def manifest_path_override(self): @property def mode(self): return self._mode + + @property + def functions_to_build(self): + if self._function_identifier: + function = self._function_provider.get(self._function_identifier) + + if not function: + all_functions = [f.name for f in self._function_provider.get_all()] + available_function_message = "{} not found. Possible options in your template: {}" \ + .format(self._function_identifier, all_functions) + LOG.info(available_function_message) + raise FunctionNotFound("Unable to find a Function with name '%s'", self._function_identifier) + + return [function] + + return self._function_provider.get_all() diff --git a/samcli/commands/build/command.py b/samcli/commands/build/command.py index 4c2df81fe8..6a95e65744 100644 --- a/samcli/commands/build/command.py +++ b/samcli/commands/build/command.py @@ -14,6 +14,7 @@ from samcli.lib.build.app_builder import ApplicationBuilder, BuildError, UnsupportedBuilderLibraryVersionError, \ ContainerBuildNotSupported from samcli.lib.build.workflow_config import UnsupportedRuntimeException +from samcli.local.lambdafn.exceptions import FunctionNotFound from samcli.commands._utils.template import move_template LOG = logging.getLogger(__name__) @@ -81,8 +82,10 @@ @docker_common_options @cli_framework_options @aws_creds_options +@click.argument('function_identifier', required=False) @pass_context def cli(ctx, + function_identifier, template, base_dir, build_dir, @@ -96,11 +99,12 @@ def cli(ctx, mode = _get_mode_value_from_envvar("SAM_BUILD_MODE", choices=["debug"]) - do_cli(template, base_dir, build_dir, True, use_container, manifest, docker_network, + do_cli(function_identifier, template, base_dir, build_dir, True, use_container, manifest, docker_network, skip_pull_image, parameter_overrides, mode) # pragma: no cover -def do_cli(template, # pylint: disable=too-many-locals +def do_cli(function_identifier, # pylint: disable=too-many-locals + template, base_dir, build_dir, clean, @@ -119,7 +123,8 @@ def do_cli(template, # pylint: disable=too-many-locals if use_container: LOG.info("Starting Build inside a container") - with BuildContext(template, + with BuildContext(function_identifier, + template, base_dir, build_dir, clean=clean, @@ -129,14 +134,16 @@ def do_cli(template, # pylint: disable=too-many-locals docker_network=docker_network, skip_pull_image=skip_pull_image, mode=mode) as ctx: + try: + builder = ApplicationBuilder(ctx.functions_to_build, + ctx.build_dir, + ctx.base_dir, + manifest_path_override=ctx.manifest_path_override, + container_manager=ctx.container_manager, + mode=ctx.mode) + except FunctionNotFound as ex: + raise UserException(str(ex)) - builder = ApplicationBuilder(ctx.function_provider, - ctx.build_dir, - ctx.base_dir, - manifest_path_override=ctx.manifest_path_override, - container_manager=ctx.container_manager, - mode=ctx.mode - ) try: artifacts = builder.build() modified_template = builder.update_template(ctx.template_dict, diff --git a/samcli/lib/build/app_builder.py b/samcli/lib/build/app_builder.py index 2ad2fe7b89..3ca2056769 100644 --- a/samcli/lib/build/app_builder.py +++ b/samcli/lib/build/app_builder.py @@ -49,7 +49,7 @@ class ApplicationBuilder(object): """ def __init__(self, - function_provider, + functions_to_build, build_dir, base_dir, manifest_path_override=None, @@ -61,8 +61,8 @@ def __init__(self, Parameters ---------- - function_provider : samcli.commands.local.lib.sam_function_provider.SamFunctionProvider - Provider that can vend out functions available in the SAM template + functions_to_build: Iterator + Iterator that can vend out functions available in the SAM template build_dir : str Path to the directory where we will be storing built artifacts @@ -79,7 +79,7 @@ def __init__(self, mode : str Optional, name of the build mode to use ex: 'debug' """ - self._function_provider = function_provider + self._functions_to_build = functions_to_build self._build_dir = build_dir self._base_dir = base_dir self._manifest_path_override = manifest_path_override @@ -100,7 +100,7 @@ def build(self): result = {} - for lambda_function in self._function_provider.get_all(): + for lambda_function in self._functions_to_build: LOG.info("Building resource '%s'", lambda_function.name) result[lambda_function.name] = self._build_function(lambda_function.name, diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index 5dd24e1d7c..703e7ff95d 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -20,6 +20,7 @@ class BuildIntegBase(TestCase): + template = "template.yaml" @classmethod def setUpClass(cls): @@ -34,7 +35,7 @@ def setUpClass(cls): cls.scratch_dir = str(Path(__file__).resolve().parent.joinpath("scratch")) cls.test_data_path = str(Path(integration_dir, "testdata", "buildcmd")) - cls.template_path = str(Path(cls.test_data_path, "template.yaml")) + cls.template_path = str(Path(cls.test_data_path, cls.template)) def setUp(self): @@ -61,9 +62,14 @@ def base_command(cls): return command def get_command_list(self, build_dir=None, base_dir=None, manifest_path=None, use_container=None, - parameter_overrides=None, mode=None): + parameter_overrides=None, mode=None, function_identifier=None): - command_list = [self.cmd, "build", "-t", self.template_path] + command_list = [self.cmd, "build"] + + if function_identifier: + command_list += [function_identifier] + + command_list += ["-t", self.template_path] if parameter_overrides: command_list += ["--parameter-overrides", self._make_parameter_override_arg(parameter_overrides)] diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index a1856b3f13..22e2876577 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -402,3 +402,83 @@ def _verify_built_artifact(self, build_dir, function_logical_id, expected_files) all_artifacts = set(os.listdir(str(resource_artifact_dir))) actual_files = all_artifacts.intersection(expected_files) self.assertEquals(actual_files, expected_files) + + +class TestBuildCommand_SingleFunctionBuilds(BuildIntegBase): + template = "many-functions-template.yaml" + + EXPECTED_FILES_GLOBAL_MANIFEST = set() + EXPECTED_FILES_PROJECT_MANIFEST = {'__init__.py', 'main.py', 'numpy', + # 'cryptography', + "jinja2", + 'requirements.txt'} + + def test_fucntion_not_found(self): + overrides = {"Runtime": 'python3.7', "CodeUri": "Python", "Handler": "main.handler"} + cmdlist = self.get_command_list(parameter_overrides=overrides, + function_identifier="FunctionNotInTemplate") + + process = subprocess.Popen(cmdlist, cwd=self.working_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + + self.assertEquals(process.returncode, 1) + self.assertIn('FunctionNotInTemplate not found', str(stderr.decode('utf8'))) + + @parameterized.expand([ + ("python3.7", False, "FunctionOne"), + ("python3.7", "use_container", "FunctionOne"), + ("python3.7", False, "FunctionTwo"), + ("python3.7", "use_container", "FunctionTwo") + ]) + def test_build_single_function(self, runtime, use_container, function_identifier): + # Don't run test on wrong Python versions + py_version = self._get_python_version() + if py_version != runtime: + self.skipTest("Current Python version '{}' does not match Lambda runtime version '{}'".format(py_version, + runtime)) + + overrides = {"Runtime": runtime, "CodeUri": "Python", "Handler": "main.handler"} + cmdlist = self.get_command_list(use_container=use_container, + parameter_overrides=overrides, + function_identifier=function_identifier) + + LOG.info("Running Command: {}", cmdlist) + process = subprocess.Popen(cmdlist, cwd=self.working_dir) + process.wait() + + self._verify_built_artifact(self.default_build_dir, function_identifier, + self.EXPECTED_FILES_PROJECT_MANIFEST) + + expected = { + "pi": "3.14", + "jinja": "Hello World" + } + self._verify_invoke_built_function(self.built_template, + function_identifier, + self._make_parameter_override_arg(overrides), + expected) + self.verify_docker_container_cleanedup(runtime) + + def _verify_built_artifact(self, build_dir, function_logical_id, expected_files): + self.assertTrue(build_dir.exists(), "Build directory should be created") + + build_dir_files = os.listdir(str(build_dir)) + self.assertIn("template.yaml", build_dir_files) + self.assertIn(function_logical_id, build_dir_files) + + template_path = build_dir.joinpath("template.yaml") + resource_artifact_dir = build_dir.joinpath(function_logical_id) + + # Make sure the template has correct CodeUri for resource + self._verify_resource_property(str(template_path), + function_logical_id, + "CodeUri", + function_logical_id) + + all_artifacts = set(os.listdir(str(resource_artifact_dir))) + print(all_artifacts) + actual_files = all_artifacts.intersection(expected_files) + self.assertEquals(actual_files, expected_files) + + def _get_python_version(self): + return "python{}.{}".format(sys.version_info.major, sys.version_info.minor) diff --git a/tests/integration/testdata/buildcmd/many-functions-template.yaml b/tests/integration/testdata/buildcmd/many-functions-template.yaml new file mode 100644 index 0000000000..1ef004b02d --- /dev/null +++ b/tests/integration/testdata/buildcmd/many-functions-template.yaml @@ -0,0 +1,28 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Parameteres: + Runtime: + Type: String + CodeUri: + Type: String + Handler: + Type: String + +Resources: + + FunctionOne: + Type: AWS::Serverless::Function + Properties: + Handler: !Ref Handler + Runtime: !Ref Runtime + CodeUri: !Ref CodeUri + Timeout: 600 + + FunctionTwo: + Type: AWS::Serverless::Function + Properties: + Handler: !Ref Handler + Runtime: !Ref Runtime + CodeUri: !Ref CodeUri + Timeout: 600 diff --git a/tests/unit/commands/buildcmd/test_build_context.py b/tests/unit/commands/buildcmd/test_build_context.py index ce9e413cef..cf68e2ed9d 100644 --- a/tests/unit/commands/buildcmd/test_build_context.py +++ b/tests/unit/commands/buildcmd/test_build_context.py @@ -15,11 +15,14 @@ def test_must_setup_context(self, ContainerManagerMock, pathlib_mock, SamFunctio get_template_data_mock): template_dict = get_template_data_mock.return_value = "template dict" - funcprovider = SamFunctionProviderMock.return_value = "funcprovider" + func_provider_mock = Mock() + func_provider_mock.get.return_value = "function to build" + funcprovider = SamFunctionProviderMock.return_value = func_provider_mock base_dir = pathlib_mock.Path.return_value.resolve.return_value.parent = "basedir" container_mgr_mock = ContainerManagerMock.return_value = Mock() - context = BuildContext("template_file", + context = BuildContext("function_identifier", + "template_file", None, # No base dir is provided "build_dir", manifest_path="manifest_path", @@ -46,6 +49,7 @@ def test_must_setup_context(self, ContainerManagerMock, pathlib_mock, SamFunctio self.assertEquals(context.output_template_path, os.path.join(build_dir_result, "template.yaml")) self.assertEquals(context.manifest_path_override, os.path.abspath("manifest_path")) self.assertEqual(context.mode, "buildmode") + self.assertEquals(context.functions_to_build, ["function to build"]) get_template_data_mock.assert_called_once_with("template_file") SamFunctionProviderMock.assert_called_once_with(template_dict, "overrides") @@ -53,6 +57,58 @@ def test_must_setup_context(self, ContainerManagerMock, pathlib_mock, SamFunctio setup_build_dir_mock.assert_called_with("build_dir", True) ContainerManagerMock.assert_called_once_with(docker_network_id="network", skip_pull_image=True) + func_provider_mock.get.assert_called_once_with("function_identifier") + + @patch("samcli.commands.build.build_context.get_template_data") + @patch("samcli.commands.build.build_context.SamFunctionProvider") + @patch("samcli.commands.build.build_context.pathlib") + @patch("samcli.commands.build.build_context.ContainerManager") + def test_must_return_many_functions_to_build(self, ContainerManagerMock, pathlib_mock, SamFunctionProviderMock, + get_template_data_mock): + template_dict = get_template_data_mock.return_value = "template dict" + func_provider_mock = Mock() + func_provider_mock.get_all.return_value = ["function to build", "and another function"] + funcprovider = SamFunctionProviderMock.return_value = func_provider_mock + base_dir = pathlib_mock.Path.return_value.resolve.return_value.parent = "basedir" + container_mgr_mock = ContainerManagerMock.return_value = Mock() + + context = BuildContext(None, + "template_file", + None, # No base dir is provided + "build_dir", + manifest_path="manifest_path", + clean=True, + use_container=True, + docker_network="network", + parameter_overrides="overrides", + skip_pull_image=True, + mode="buildmode") + setup_build_dir_mock = Mock() + build_dir_result = setup_build_dir_mock.return_value = "my/new/build/dir" + context._setup_build_dir = setup_build_dir_mock + + # call the enter method + result = context.__enter__() + + self.assertEquals(result, context) # __enter__ must return self + self.assertEquals(context.template_dict, template_dict) + self.assertEquals(context.function_provider, funcprovider) + self.assertEquals(context.base_dir, base_dir) + self.assertEquals(context.container_manager, container_mgr_mock) + self.assertEquals(context.build_dir, build_dir_result) + self.assertEquals(context.use_container, True) + self.assertEquals(context.output_template_path, os.path.join(build_dir_result, "template.yaml")) + self.assertEquals(context.manifest_path_override, os.path.abspath("manifest_path")) + self.assertEqual(context.mode, "buildmode") + self.assertEquals(context.functions_to_build, ["function to build", "and another function"]) + + get_template_data_mock.assert_called_once_with("template_file") + SamFunctionProviderMock.assert_called_once_with(template_dict, "overrides") + pathlib_mock.Path.assert_called_once_with("template_file") + setup_build_dir_mock.assert_called_with("build_dir", True) + ContainerManagerMock.assert_called_once_with(docker_network_id="network", + skip_pull_image=True) + func_provider_mock.get_all.assert_called_once() class TestBuildContext_setup_build_dir(TestCase): diff --git a/tests/unit/commands/buildcmd/test_command.py b/tests/unit/commands/buildcmd/test_command.py index 8b867fe480..1df0bbf946 100644 --- a/tests/unit/commands/buildcmd/test_command.py +++ b/tests/unit/commands/buildcmd/test_command.py @@ -9,6 +9,7 @@ from samcli.commands.exceptions import UserException from samcli.lib.build.app_builder import BuildError, UnsupportedBuilderLibraryVersionError from samcli.lib.build.workflow_config import UnsupportedRuntimeException +from samcli.local.lambdafn.exceptions import FunctionNotFound class TestDoCli(TestCase): @@ -30,10 +31,10 @@ def test_must_succeed_build(self, artifacts = builder_mock.build.return_value = "artifacts" modified_template = builder_mock.update_template.return_value = "modified template" - do_cli("template", "base_dir", "build_dir", "clean", "use_container", + do_cli("function_identifier", "template", "base_dir", "build_dir", "clean", "use_container", "manifest_path", "docker_network", "skip_pull", "parameter_overrides", "mode") - ApplicationBuilderMock.assert_called_once_with(ctx_mock.function_provider, + ApplicationBuilderMock.assert_called_once_with(ctx_mock.functions_to_build, ctx_mock.build_dir, ctx_mock.base_dir, manifest_path_override=ctx_mock.manifest_path_override, @@ -64,11 +65,25 @@ def test_must_catch_known_exceptions(self, exception, ApplicationBuilderMock, Bu builder_mock.build.side_effect = exception with self.assertRaises(UserException) as ctx: - do_cli("template", "base_dir", "build_dir", "clean", "use_container", + do_cli("function_identifier", "template", "base_dir", "build_dir", "clean", "use_container", "manifest_path", "docker_network", "skip_pull", "parameteroverrides", "mode") self.assertEquals(str(ctx.exception), str(exception)) + @patch("samcli.commands.build.command.BuildContext") + @patch("samcli.commands.build.command.ApplicationBuilder") + def test_must_catch_function_not_found_exception(self, ApplicationBuilderMock, BuildContextMock): + ctx_mock = Mock() + BuildContextMock.return_value.__enter__ = Mock() + BuildContextMock.return_value.__enter__.return_value = ctx_mock + ApplicationBuilderMock.side_effect = FunctionNotFound('Function Not Found') + + with self.assertRaises(UserException) as ctx: + do_cli("function_identifier", "template", "base_dir", "build_dir", "clean", "use_container", + "manifest_path", "docker_network", "skip_pull", "parameteroverrides", "mode") + + self.assertEquals(str(ctx.exception), 'Function Not Found') + class TestGetModeValueFromEnvvar(TestCase): diff --git a/tests/unit/lib/build_module/test_app_builder.py b/tests/unit/lib/build_module/test_app_builder.py index a9c628f00d..8450af2e51 100644 --- a/tests/unit/lib/build_module/test_app_builder.py +++ b/tests/unit/lib/build_module/test_app_builder.py @@ -14,29 +14,27 @@ class TestApplicationBuilder_build(TestCase): def setUp(self): - self.mock_func_provider = Mock() - self.builder = ApplicationBuilder(self.mock_func_provider, + self.func1 = Mock() + self.func2 = Mock() + self.builder = ApplicationBuilder([self.func1, self.func2], "builddir", "basedir") def test_must_iterate_on_functions(self): - func1 = Mock() - func2 = Mock() build_function_mock = Mock() - self.mock_func_provider.get_all.return_value = [func1, func2] self.builder._build_function = build_function_mock result = self.builder.build() self.assertEquals(result, { - func1.name: build_function_mock.return_value, - func2.name: build_function_mock.return_value, + self.func1.name: build_function_mock.return_value, + self.func2.name: build_function_mock.return_value, }) build_function_mock.assert_has_calls([ - call(func1.name, func1.codeuri, func1.runtime), - call(func2.name, func2.codeuri, func2.runtime), + call(self.func1.name, self.func1.codeuri, self.func1.runtime), + call(self.func2.name, self.func2.codeuri, self.func2.runtime), ], any_order=False) From d7c6ce6ef9b7df9933daca58d04d98ba69fca75a Mon Sep 17 00:00:00 2001 From: Jacob Fuss Date: Wed, 24 Apr 2019 07:38:03 -0700 Subject: [PATCH 2/2] Remove prints and fix spelling mistakes --- tests/integration/buildcmd/test_build_cmd.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index 22e2876577..c180c32be2 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -86,7 +86,6 @@ def _verify_built_artifact(self, build_dir, function_logical_id, expected_files) function_logical_id) all_artifacts = set(os.listdir(str(resource_artifact_dir))) - print(all_artifacts) actual_files = all_artifacts.intersection(expected_files) self.assertEquals(actual_files, expected_files) @@ -413,7 +412,7 @@ class TestBuildCommand_SingleFunctionBuilds(BuildIntegBase): "jinja2", 'requirements.txt'} - def test_fucntion_not_found(self): + def test_function_not_found(self): overrides = {"Runtime": 'python3.7', "CodeUri": "Python", "Handler": "main.handler"} cmdlist = self.get_command_list(parameter_overrides=overrides, function_identifier="FunctionNotInTemplate") @@ -476,7 +475,6 @@ def _verify_built_artifact(self, build_dir, function_logical_id, expected_files) function_logical_id) all_artifacts = set(os.listdir(str(resource_artifact_dir))) - print(all_artifacts) actual_files = all_artifacts.intersection(expected_files) self.assertEquals(actual_files, expected_files)