From c15c6b0b0bc3e9773ac7881b396534c3d5a9c0aa Mon Sep 17 00:00:00 2001 From: Volkan Gurel Date: Thu, 27 Dec 2018 16:44:39 -0800 Subject: [PATCH 1/6] Add the Go Modules builder Add a builder for Go that assumes the given go project uses Go Modules, and so can be built using standard go tooling. --- aws_lambda_builders/actions.py | 10 ++- aws_lambda_builders/workflows/__init__.py | 1 + .../workflows/go_modules/DESIGN.md | 53 +++++++++++++++ .../workflows/go_modules/__init__.py | 5 ++ .../workflows/go_modules/actions.py | 29 ++++++++ .../workflows/go_modules/builder.py | 67 +++++++++++++++++++ .../workflows/go_modules/utils.py | 30 +++++++++ .../workflows/go_modules/workflow.py | 45 +++++++++++++ .../workflows/go_modules/test_go_utils.py | 43 ++++++++++++ .../workflows/go_modules/test_go.py | 52 ++++++++++++++ .../go_modules/testdata/broken-deps/go.mod | 12 ++++ .../go_modules/testdata/broken-deps/main.go | 4 ++ .../go_modules/testdata/no-deps/go.mod | 1 + .../go_modules/testdata/no-deps/main.go | 4 ++ .../go_modules/testdata/with-deps/go.mod | 12 ++++ .../go_modules/testdata/with-deps/go.sum | 10 +++ .../go_modules/testdata/with-deps/main.go | 17 +++++ tests/unit/workflows/go_modules/__init__.py | 0 .../unit/workflows/go_modules/test_actions.py | 24 +++++++ .../unit/workflows/go_modules/test_builder.py | 50 ++++++++++++++ .../workflows/go_modules/test_workflow.py | 18 +++++ 21 files changed, 485 insertions(+), 2 deletions(-) create mode 100644 aws_lambda_builders/workflows/go_modules/DESIGN.md create mode 100644 aws_lambda_builders/workflows/go_modules/__init__.py create mode 100644 aws_lambda_builders/workflows/go_modules/actions.py create mode 100644 aws_lambda_builders/workflows/go_modules/builder.py create mode 100644 aws_lambda_builders/workflows/go_modules/utils.py create mode 100644 aws_lambda_builders/workflows/go_modules/workflow.py create mode 100644 tests/functional/workflows/go_modules/test_go_utils.py create mode 100644 tests/integration/workflows/go_modules/test_go.py create mode 100644 tests/integration/workflows/go_modules/testdata/broken-deps/go.mod create mode 100644 tests/integration/workflows/go_modules/testdata/broken-deps/main.go create mode 100644 tests/integration/workflows/go_modules/testdata/no-deps/go.mod create mode 100644 tests/integration/workflows/go_modules/testdata/no-deps/main.go create mode 100644 tests/integration/workflows/go_modules/testdata/with-deps/go.mod create mode 100644 tests/integration/workflows/go_modules/testdata/with-deps/go.sum create mode 100644 tests/integration/workflows/go_modules/testdata/with-deps/main.go create mode 100644 tests/unit/workflows/go_modules/__init__.py create mode 100644 tests/unit/workflows/go_modules/test_actions.py create mode 100644 tests/unit/workflows/go_modules/test_builder.py create mode 100644 tests/unit/workflows/go_modules/test_workflow.py diff --git a/aws_lambda_builders/actions.py b/aws_lambda_builders/actions.py index d37d361c1..78e4e9bee 100644 --- a/aws_lambda_builders/actions.py +++ b/aws_lambda_builders/actions.py @@ -92,10 +92,16 @@ class CopySourceAction(BaseAction): PURPOSE = Purpose.COPY_SOURCE - def __init__(self, source_dir, dest_dir, excludes=None): + def __init__(self, source_dir, dest_dir, only=None, excludes=None): self.source_dir = source_dir self.dest_dir = dest_dir + self.only = only self.excludes = excludes or [] def execute(self): - copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes)) + if self.only: + def ignore(source, names): + return [name for name in names if name not in self.only] + else: + ignore = shutil.ignore_patterns(*self.excludes) + copytree(self.source_dir, self.dest_dir, ignore=ignore) diff --git a/aws_lambda_builders/workflows/__init__.py b/aws_lambda_builders/workflows/__init__.py index 8e8425446..24b82710a 100644 --- a/aws_lambda_builders/workflows/__init__.py +++ b/aws_lambda_builders/workflows/__init__.py @@ -6,3 +6,4 @@ import aws_lambda_builders.workflows.nodejs_npm import aws_lambda_builders.workflows.ruby_bundler import aws_lambda_builders.workflows.go_dep +import aws_lambda_builders.workflows.go_modules diff --git a/aws_lambda_builders/workflows/go_modules/DESIGN.md b/aws_lambda_builders/workflows/go_modules/DESIGN.md new file mode 100644 index 000000000..7e159273b --- /dev/null +++ b/aws_lambda_builders/workflows/go_modules/DESIGN.md @@ -0,0 +1,53 @@ +## Go - Go Modules Lambda Builder + +### Scope + +This package leverages standard Go tooling available as of Go1.11 to build Go +applications to be deployed in an AWS Lambda environment. The scope of this +builder is to take an existing directory containing customer code, and a +top-level `go.mod` file specifying third party dependencies. The builder will +run `go build` on the project and put the resulting binary in the given +artifacts folder. + +### Interface + +The top level interface is presented by the `GoModulesBuilder` class. There +will be one public method `build`, which takes the provided arguments and +builds a static binary using standard go tools. + +```python +def build(self, source_dir_path, artifacts_dir_path, ui=None, config=None): + """Builds a go project into an artifact directory. + + :type source_dir_path: str + :param source_dir_path: Directory with the source files. + + :type artifacts_dir_path: str + :param artifacts_dir_path: Directory to write dependencies into. + + :type executable_name: str + :param executable_name: Name of the executable to create from the build. + + :type ui: :class:`lambda_builders.utils.UI` or None + :param ui: A class that traps all progress information such as status + and errors. If injected by the caller, it can be used to monitor + the status of the build process or forward this information + elsewhere. + + :type config: :class:`lambda_builders.utils.Config` or None + :param config: To be determined. This is an optional config object + we can extend at a later date to add more options to how modules is + called. + """ +``` + +### Implementation + +The general algorithm for preparing a Go package for use on AWS Lambda +is very simple. It's as follows: + +Pass in GOOS=linux and GOARCH=amd64 to the `go build` command to target the +OS and architecture used on AWS Lambda. Let go tooling handle the +cross-compilation, regardless of the build environment. Move the resulting +static binary to the artifacts folder to be shipped as a single-file zip +archive. diff --git a/aws_lambda_builders/workflows/go_modules/__init__.py b/aws_lambda_builders/workflows/go_modules/__init__.py new file mode 100644 index 000000000..a46dddfab --- /dev/null +++ b/aws_lambda_builders/workflows/go_modules/__init__.py @@ -0,0 +1,5 @@ +""" +Builds Go Lambda functions using standard Go tooling +""" + +from .workflow import GoModulesWorkflow diff --git a/aws_lambda_builders/workflows/go_modules/actions.py b/aws_lambda_builders/workflows/go_modules/actions.py new file mode 100644 index 000000000..0a713ca7c --- /dev/null +++ b/aws_lambda_builders/workflows/go_modules/actions.py @@ -0,0 +1,29 @@ +""" +Action to build a Go project using standard Go tooling +""" + +from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError +from .builder import BuilderError + + +class GoModulesBuildAction(BaseAction): + + NAME = "Build" + DESCRIPTION = "Building Go package with Go Modules" + PURPOSE = Purpose.RESOLVE_DEPENDENCIES + + def __init__(self, source_dir, artifacts_dir, executable_name, builder): + self.source_dir = source_dir + self.artifacts_dir = artifacts_dir + self.executable_name = executable_name + self.builder = builder + + def execute(self): + try: + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.executable_name + ) + except BuilderError as ex: + raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/go_modules/builder.py b/aws_lambda_builders/workflows/go_modules/builder.py new file mode 100644 index 000000000..1391768ac --- /dev/null +++ b/aws_lambda_builders/workflows/go_modules/builder.py @@ -0,0 +1,67 @@ +""" +Build a Go project using standard Go tooling +""" +import logging + + +LOG = logging.getLogger(__name__) + + +class BuilderError(Exception): + MESSAGE = "Builder Failed: {message}" + + def __init__(self, **kwargs): + Exception.__init__(self, self.MESSAGE.format(**kwargs)) + + +class GoModulesBuilder(object): + def __init__(self, osutils): + """Initialize a GoModulesBuilder. + + :type osutils: :class:`lambda_builders.utils.OSUtils` + :param osutils: A class used for all interactions with the + outside OS. + """ + self.osutils = osutils + + def build(self, source_dir_path, artifacts_dir_path, executable_name): + """Builds a go project into an artifact directory. + + :type source_dir_path: str + :param source_dir_path: Directory with the source files. + + :type artifacts_dir_path: str + :param artifacts_dir_path: Directory to write dependencies into. + + :type executable_name: str + :param executable_name: Name of the executable to create from the build. + + :type ui: :class:`lambda_builders.utils.UI` or None + :param ui: A class that traps all progress information such as status + and errors. If injected by the caller, it can be used to monitor + the status of the build process or forward this information + elsewhere. + + :type config: :class:`lambda_builders.utils.Config` or None + :param config: To be determined. This is an optional config object + we can extend at a later date to add more options to how modules is + called. + """ + env = {} + env.update(self.osutils.environ) + env.update({"GOOS": "linux", "GOARCH": "amd64"}) + cmd = ["go", "build", "-o", self.osutils.joinpath(artifacts_dir_path, executable_name), source_dir_path] + + p = self.osutils.popen( + cmd, + cwd=source_dir_path, + env=env, + stdout=self.osutils.pipe, + stderr=self.osutils.pipe, + ) + out, err = p.communicate() + + if p.returncode != 0: + raise BuilderError(message=err.decode("utf8").strip()) + + return out.decode("utf8").strip() diff --git a/aws_lambda_builders/workflows/go_modules/utils.py b/aws_lambda_builders/workflows/go_modules/utils.py new file mode 100644 index 000000000..9c637cbcf --- /dev/null +++ b/aws_lambda_builders/workflows/go_modules/utils.py @@ -0,0 +1,30 @@ +""" +Commonly used utilities +""" + +import os +import subprocess + + +class OSUtils(object): + """ + Wrapper around file system functions, to make it easy to + unit test actions in memory + """ + @property + def environ(self): + return os.environ + + def joinpath(self, *args): + return os.path.join(*args) + + def basename(self, *args): + return os.path.basename(*args) + + def popen(self, command, stdout=None, stderr=None, env=None, cwd=None): + p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd) + return p + + @property + def pipe(self): + return subprocess.PIPE diff --git a/aws_lambda_builders/workflows/go_modules/workflow.py b/aws_lambda_builders/workflows/go_modules/workflow.py new file mode 100644 index 000000000..67fd03811 --- /dev/null +++ b/aws_lambda_builders/workflows/go_modules/workflow.py @@ -0,0 +1,45 @@ +""" +Go Modules Workflow +""" +from aws_lambda_builders.workflow import BaseWorkflow, Capability +from aws_lambda_builders.actions import CopySourceAction + +from .actions import GoModulesBuildAction +from .builder import GoModulesBuilder +from .utils import OSUtils + + +class GoModulesWorkflow(BaseWorkflow): + + NAME = "GoModulesBuilder" + + CAPABILITY = Capability(language="go", + dependency_manager="modules", + application_framework=None) + + def __init__(self, + source_dir, + artifacts_dir, + scratch_dir, + manifest_path, + runtime=None, + osutils=None, + **kwargs): + + super(GoModulesWorkflow, self).__init__( + source_dir, + artifacts_dir, + scratch_dir, + manifest_path, + runtime=runtime, + **kwargs) + + if osutils is None: + osutils = OSUtils() + + executable_name = osutils.basename(source_dir) + builder = GoModulesBuilder(osutils) + self.actions = [ + GoModulesBuildAction(source_dir, artifacts_dir, executable_name, builder), + CopySourceAction(source_dir, artifacts_dir, only=[executable_name]), + ] diff --git a/tests/functional/workflows/go_modules/test_go_utils.py b/tests/functional/workflows/go_modules/test_go_utils.py new file mode 100644 index 000000000..1581f949e --- /dev/null +++ b/tests/functional/workflows/go_modules/test_go_utils.py @@ -0,0 +1,43 @@ +import os +import sys + +from unittest import TestCase + +from aws_lambda_builders.workflows.go_modules import utils + + +class TestOSUtils(TestCase): + + def setUp(self): + self.osutils = utils.OSUtils() + + def test_environ_returns_environment(self): + result = self.osutils.environ + self.assertEqual(result, os.environ) + + def test_joinpath_joins_path_components(self): + result = self.osutils.joinpath('a', 'b', 'c') + self.assertEqual(result, os.path.join('a', 'b', 'c')) + + def test_basename_returns_path_basename(self): + result = self.osutils.basename(os.path.dirname(__file__)) + self.assertEqual(result, 'go_modules') + + def test_popen_runs_a_process_and_returns_outcome(self): + cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py') + p = self.osutils.popen([sys.executable, cwd_py], + stdout=self.osutils.pipe, + stderr=self.osutils.pipe) + out, err = p.communicate() + self.assertEqual(p.returncode, 0) + self.assertEqual(out.decode('utf8').strip(), os.getcwd()) + + def test_popen_can_accept_cwd(self): + testdata_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata') + p = self.osutils.popen([sys.executable, 'cwd.py'], + stdout=self.osutils.pipe, + stderr=self.osutils.pipe, + cwd=testdata_dir) + out, err = p.communicate() + self.assertEqual(p.returncode, 0) + self.assertEqual(out.decode('utf8').strip(), os.path.abspath(testdata_dir)) diff --git a/tests/integration/workflows/go_modules/test_go.py b/tests/integration/workflows/go_modules/test_go.py new file mode 100644 index 000000000..7d3af7b1a --- /dev/null +++ b/tests/integration/workflows/go_modules/test_go.py @@ -0,0 +1,52 @@ +import os +import shutil +import tempfile + +from unittest import TestCase + +from aws_lambda_builders.builder import LambdaBuilder +from aws_lambda_builders.exceptions import WorkflowFailedError + + +class TestGoWorkflow(TestCase): + """ + Verifies that `go` workflow works by building a Lambda using Go Modules + """ + + TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata") + + def setUp(self): + self.artifacts_dir = tempfile.mkdtemp() + self.scratch_dir = tempfile.mkdtemp() + self.builder = LambdaBuilder(language="go", + dependency_manager="modules", + application_framework=None) + + def tearDown(self): + shutil.rmtree(self.artifacts_dir) + shutil.rmtree(self.scratch_dir) + + def test_builds_project_without_dependencies(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") + self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, + os.path.join(source_dir, "go.mod")) + expected_files = {"no-deps"} + output_files = set(os.listdir(self.artifacts_dir)) + print(output_files) + self.assertEquals(expected_files, output_files) + + def test_builds_project_with_dependencies(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps") + self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, + os.path.join(source_dir, "go.mod")) + expected_files = {"with-deps"} + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEquals(expected_files, output_files) + + def test_fails_if_modules_cannot_resolve_dependencies(self): + source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps") + with self.assertRaises(WorkflowFailedError) as ctx: + self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, + os.path.join(source_dir, "go.mod")) + self.assertIn("GoModulesBuilder:Build - Builder Failed: ", + str(ctx.exception)) diff --git a/tests/integration/workflows/go_modules/testdata/broken-deps/go.mod b/tests/integration/workflows/go_modules/testdata/broken-deps/go.mod new file mode 100644 index 000000000..864baf723 --- /dev/null +++ b/tests/integration/workflows/go_modules/testdata/broken-deps/go.mod @@ -0,0 +1,12 @@ +module github.com/awslabs/aws-lambda-builders + +require ( + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/aws/aws-lambda-go v0.9999.0 // doesn't exist, broken dependency + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.1.1 // indirect + github.com/stretchr/testify v1.2.2 // indirect + gopkg.in/urfave/cli.v1 v1.20.0 // indirect + gopkg.in/yaml.v2 v2.2.2 // indirect +) diff --git a/tests/integration/workflows/go_modules/testdata/broken-deps/main.go b/tests/integration/workflows/go_modules/testdata/broken-deps/main.go new file mode 100644 index 000000000..da29a2cad --- /dev/null +++ b/tests/integration/workflows/go_modules/testdata/broken-deps/main.go @@ -0,0 +1,4 @@ +package main + +func main() { +} diff --git a/tests/integration/workflows/go_modules/testdata/no-deps/go.mod b/tests/integration/workflows/go_modules/testdata/no-deps/go.mod new file mode 100644 index 000000000..846a0e06b --- /dev/null +++ b/tests/integration/workflows/go_modules/testdata/no-deps/go.mod @@ -0,0 +1 @@ +module github.com/awslabs/aws-lambda-builders diff --git a/tests/integration/workflows/go_modules/testdata/no-deps/main.go b/tests/integration/workflows/go_modules/testdata/no-deps/main.go new file mode 100644 index 000000000..da29a2cad --- /dev/null +++ b/tests/integration/workflows/go_modules/testdata/no-deps/main.go @@ -0,0 +1,4 @@ +package main + +func main() { +} diff --git a/tests/integration/workflows/go_modules/testdata/with-deps/go.mod b/tests/integration/workflows/go_modules/testdata/with-deps/go.mod new file mode 100644 index 000000000..18f6e0f05 --- /dev/null +++ b/tests/integration/workflows/go_modules/testdata/with-deps/go.mod @@ -0,0 +1,12 @@ +module github.com/awslabs/aws-lambda-builders + +require ( + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/aws/aws-lambda-go v1.8.0 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.1.1 // indirect + github.com/stretchr/testify v1.2.2 // indirect + gopkg.in/urfave/cli.v1 v1.20.0 // indirect + gopkg.in/yaml.v2 v2.2.2 // indirect +) diff --git a/tests/integration/workflows/go_modules/testdata/with-deps/go.sum b/tests/integration/workflows/go_modules/testdata/with-deps/go.sum new file mode 100644 index 000000000..2a45ed3a9 --- /dev/null +++ b/tests/integration/workflows/go_modules/testdata/with-deps/go.sum @@ -0,0 +1,10 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aws/aws-lambda-go v1.8.0 h1:YMCzi9FP7MNVVj9AkGpYyaqh/mvFOjhqiDtnNlWtKTg= +github.com/aws/aws-lambda-go v1.8.0/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/tests/integration/workflows/go_modules/testdata/with-deps/main.go b/tests/integration/workflows/go_modules/testdata/with-deps/main.go new file mode 100644 index 000000000..a119afd60 --- /dev/null +++ b/tests/integration/workflows/go_modules/testdata/with-deps/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-lambda-go/lambda" +) + +func handler(request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { + return events.APIGatewayProxyResponse{ + Body: "Hello World", + StatusCode: 200, + }, nil +} + +func main() { + lambda.Start(handler) +} diff --git a/tests/unit/workflows/go_modules/__init__.py b/tests/unit/workflows/go_modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/workflows/go_modules/test_actions.py b/tests/unit/workflows/go_modules/test_actions.py new file mode 100644 index 000000000..cbb7c2bd9 --- /dev/null +++ b/tests/unit/workflows/go_modules/test_actions.py @@ -0,0 +1,24 @@ +from unittest import TestCase +from mock import patch + +from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.workflows.go_modules.actions import GoModulesBuildAction +from aws_lambda_builders.workflows.go_modules.builder import BuilderError + + +class TestGoModulesBuildAction(TestCase): + @patch("aws_lambda_builders.workflows.go_modules.builder.GoModulesBuilder") + def test_runs_bundle_install(self, BuilderMock): + builder = BuilderMock.return_value + action = GoModulesBuildAction("source_dir", "artifacts_dir", "executable_name", builder) + action.execute() + builder.build.assert_called_with("source_dir", "artifacts_dir", "executable_name") + + @patch("aws_lambda_builders.workflows.go_modules.builder.GoModulesBuilder") + def test_raises_action_failed_on_failure(self, BuilderMock): + builder = BuilderMock.return_value + builder.build.side_effect = BuilderError(message="Fail") + action = GoModulesBuildAction("source_dir", "artifacts_dir", "executable_name", builder) + with self.assertRaises(ActionFailedError) as raised: + action.execute() + self.assertEqual(raised.exception.args[0], "Builder Failed: Fail") diff --git a/tests/unit/workflows/go_modules/test_builder.py b/tests/unit/workflows/go_modules/test_builder.py new file mode 100644 index 000000000..6587cc26f --- /dev/null +++ b/tests/unit/workflows/go_modules/test_builder.py @@ -0,0 +1,50 @@ +from unittest import TestCase +from mock import patch + +from aws_lambda_builders.workflows.go_modules.builder import GoModulesBuilder, BuilderError + + +class FakePopen: + def __init__(self, out=b'out', err=b'err', retcode=0): + self.out = out + self.err = err + self.returncode = retcode + + def communicate(self): + return self.out, self.err + + +class TestSubprocessBundler(TestCase): + + @patch("aws_lambda_builders.workflows.go_modules.utils.OSUtils") + def setUp(self, OSUtilMock): + self.osutils = OSUtilMock.return_value + self.osutils.pipe = 'PIPE' + self.popen = FakePopen() + self.osutils.popen.side_effect = [self.popen] + self.under_test = GoModulesBuilder(self.osutils) + + def test_run_executes_bundler_on_nixes(self): + self.osutils.is_windows.side_effect = [False] + self.under_test = GoModulesBuilder(self.osutils) + self.under_test.build("source_dir", "artifacts_dir", "executable_name") + self.osutils.popen.assert_called_with( + ["go", "build", "-o", + self.osutils.joinpath("artifacts_dir", "executable_name"), "source_dir"], + cwd="source_dir", + env={'GOOS': 'linux', 'GOARCH': 'amd64'}, + stderr='PIPE', + stdout='PIPE', + ) + + def test_returns_popen_out_decoded_if_retcode_is_0(self): + self.popen.out = b'some encoded text\n\n' + result = self.under_test.build("source_dir", "artifacts_dir", "executable_name") + self.assertEqual(result, 'some encoded text') + + def test_raises_BuilderError_with_err_text_if_retcode_is_not_0(self): + self.popen.returncode = 1 + self.popen.err = b'some error text\n\n' + with self.assertRaises(BuilderError) as raised: + self.under_test.build("source_dir", "artifacts_dir", "executable_name") + self.assertEqual(raised.exception.args[0], "Builder Failed: some error text") diff --git a/tests/unit/workflows/go_modules/test_workflow.py b/tests/unit/workflows/go_modules/test_workflow.py new file mode 100644 index 000000000..603386954 --- /dev/null +++ b/tests/unit/workflows/go_modules/test_workflow.py @@ -0,0 +1,18 @@ +from unittest import TestCase + +from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.workflows.go_modules.workflow import GoModulesWorkflow +from aws_lambda_builders.workflows.go_modules.actions import GoModulesBuildAction + + +class TestGoModulesWorkflow(TestCase): + """ + the workflow requires an external utility (builder) to run, so it is extensively tested in integration tests. + this is just a quick wiring test to provide fast feedback if things are badly broken + """ + + def test_workflow_sets_up_builder_actions(self): + workflow = GoModulesWorkflow("source", "artifacts", "scratch_dir", "manifest") + self.assertEqual(len(workflow.actions), 2) + self.assertIsInstance(workflow.actions[0], GoModulesBuildAction) + self.assertIsInstance(workflow.actions[1], CopySourceAction) From b76731173562e994ead543a82f9c48c63c76e251 Mon Sep 17 00:00:00 2001 From: Volkan Gurel Date: Fri, 28 Dec 2018 11:23:53 -0800 Subject: [PATCH 2/6] fix: Fix documentation for the build function in the go workflow --- aws_lambda_builders/workflows/go_modules/DESIGN.md | 14 +------------- .../workflows/go_modules/builder.py | 11 ----------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/aws_lambda_builders/workflows/go_modules/DESIGN.md b/aws_lambda_builders/workflows/go_modules/DESIGN.md index 7e159273b..e90df7e35 100644 --- a/aws_lambda_builders/workflows/go_modules/DESIGN.md +++ b/aws_lambda_builders/workflows/go_modules/DESIGN.md @@ -16,7 +16,7 @@ will be one public method `build`, which takes the provided arguments and builds a static binary using standard go tools. ```python -def build(self, source_dir_path, artifacts_dir_path, ui=None, config=None): +def build(self, source_dir_path, artifacts_dir_path, executable_name): """Builds a go project into an artifact directory. :type source_dir_path: str @@ -27,18 +27,6 @@ def build(self, source_dir_path, artifacts_dir_path, ui=None, config=None): :type executable_name: str :param executable_name: Name of the executable to create from the build. - - :type ui: :class:`lambda_builders.utils.UI` or None - :param ui: A class that traps all progress information such as status - and errors. If injected by the caller, it can be used to monitor - the status of the build process or forward this information - elsewhere. - - :type config: :class:`lambda_builders.utils.Config` or None - :param config: To be determined. This is an optional config object - we can extend at a later date to add more options to how modules is - called. - """ ``` ### Implementation diff --git a/aws_lambda_builders/workflows/go_modules/builder.py b/aws_lambda_builders/workflows/go_modules/builder.py index 1391768ac..a860d1f97 100644 --- a/aws_lambda_builders/workflows/go_modules/builder.py +++ b/aws_lambda_builders/workflows/go_modules/builder.py @@ -35,17 +35,6 @@ def build(self, source_dir_path, artifacts_dir_path, executable_name): :type executable_name: str :param executable_name: Name of the executable to create from the build. - - :type ui: :class:`lambda_builders.utils.UI` or None - :param ui: A class that traps all progress information such as status - and errors. If injected by the caller, it can be used to monitor - the status of the build process or forward this information - elsewhere. - - :type config: :class:`lambda_builders.utils.Config` or None - :param config: To be determined. This is an optional config object - we can extend at a later date to add more options to how modules is - called. """ env = {} env.update(self.osutils.environ) From dd837e0e5b9736e58984cc4fe65e4f7195fe1d10 Mon Sep 17 00:00:00 2001 From: Volkan Gurel Date: Tue, 8 Jan 2019 20:43:46 -0800 Subject: [PATCH 3/6] Address PR feedback --- aws_lambda_builders/actions.py | 10 +--- .../workflows/go_modules/DESIGN.md | 9 +-- .../workflows/go_modules/actions.py | 8 +-- .../workflows/go_modules/builder.py | 19 ++++--- .../workflows/go_modules/path_resolver.py | 25 ++++++++ .../workflows/go_modules/utils.py | 5 +- .../workflows/go_modules/validator.py | 57 +++++++++++++++++++ .../workflows/go_modules/workflow.py | 20 +++++-- requirements/base.txt | 1 + .../workflows/go_modules/test_go_utils.py | 4 -- .../workflows/go_modules/test_go.py | 17 ++++-- .../unit/workflows/go_modules/test_actions.py | 6 +- .../unit/workflows/go_modules/test_builder.py | 15 +++-- .../go_modules/test_path_resolver.py | 28 +++++++++ .../workflows/go_modules/test_validator.py | 48 ++++++++++++++++ .../workflows/go_modules/test_workflow.py | 9 +-- 16 files changed, 220 insertions(+), 61 deletions(-) create mode 100644 aws_lambda_builders/workflows/go_modules/path_resolver.py create mode 100644 aws_lambda_builders/workflows/go_modules/validator.py create mode 100644 tests/unit/workflows/go_modules/test_path_resolver.py create mode 100644 tests/unit/workflows/go_modules/test_validator.py diff --git a/aws_lambda_builders/actions.py b/aws_lambda_builders/actions.py index 78e4e9bee..d37d361c1 100644 --- a/aws_lambda_builders/actions.py +++ b/aws_lambda_builders/actions.py @@ -92,16 +92,10 @@ class CopySourceAction(BaseAction): PURPOSE = Purpose.COPY_SOURCE - def __init__(self, source_dir, dest_dir, only=None, excludes=None): + def __init__(self, source_dir, dest_dir, excludes=None): self.source_dir = source_dir self.dest_dir = dest_dir - self.only = only self.excludes = excludes or [] def execute(self): - if self.only: - def ignore(source, names): - return [name for name in names if name not in self.only] - else: - ignore = shutil.ignore_patterns(*self.excludes) - copytree(self.source_dir, self.dest_dir, ignore=ignore) + copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes)) diff --git a/aws_lambda_builders/workflows/go_modules/DESIGN.md b/aws_lambda_builders/workflows/go_modules/DESIGN.md index e90df7e35..22b0bf00c 100644 --- a/aws_lambda_builders/workflows/go_modules/DESIGN.md +++ b/aws_lambda_builders/workflows/go_modules/DESIGN.md @@ -17,16 +17,13 @@ builds a static binary using standard go tools. ```python def build(self, source_dir_path, artifacts_dir_path, executable_name): - """Builds a go project into an artifact directory. + """Builds a go project onto an output path. :type source_dir_path: str :param source_dir_path: Directory with the source files. - :type artifacts_dir_path: str - :param artifacts_dir_path: Directory to write dependencies into. - - :type executable_name: str - :param executable_name: Name of the executable to create from the build. + :type output_path: str + :param output_path: Filename to write the executable output to. ``` ### Implementation diff --git a/aws_lambda_builders/workflows/go_modules/actions.py b/aws_lambda_builders/workflows/go_modules/actions.py index 0a713ca7c..b867ddb95 100644 --- a/aws_lambda_builders/workflows/go_modules/actions.py +++ b/aws_lambda_builders/workflows/go_modules/actions.py @@ -12,18 +12,16 @@ class GoModulesBuildAction(BaseAction): DESCRIPTION = "Building Go package with Go Modules" PURPOSE = Purpose.RESOLVE_DEPENDENCIES - def __init__(self, source_dir, artifacts_dir, executable_name, builder): + def __init__(self, source_dir, output_path, builder): self.source_dir = source_dir - self.artifacts_dir = artifacts_dir - self.executable_name = executable_name + self.output_path = output_path self.builder = builder def execute(self): try: self.builder.build( self.source_dir, - self.artifacts_dir, - self.executable_name + self.output_path, ) except BuilderError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/go_modules/builder.py b/aws_lambda_builders/workflows/go_modules/builder.py index a860d1f97..07d9cb063 100644 --- a/aws_lambda_builders/workflows/go_modules/builder.py +++ b/aws_lambda_builders/workflows/go_modules/builder.py @@ -15,31 +15,32 @@ def __init__(self, **kwargs): class GoModulesBuilder(object): - def __init__(self, osutils): + def __init__(self, osutils, runtime_path): """Initialize a GoModulesBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the outside OS. + + :type runtime_path: str + :param runtime_path: The path to the go runtime. """ self.osutils = osutils + self.runtime_path = runtime_path - def build(self, source_dir_path, artifacts_dir_path, executable_name): - """Builds a go project into an artifact directory. + def build(self, source_dir_path, output_path): + """Builds a go project onto an output path. :type source_dir_path: str :param source_dir_path: Directory with the source files. - :type artifacts_dir_path: str - :param artifacts_dir_path: Directory to write dependencies into. - - :type executable_name: str - :param executable_name: Name of the executable to create from the build. + :type output_path: str + :param output_path: Filename to write the executable output to. """ env = {} env.update(self.osutils.environ) env.update({"GOOS": "linux", "GOARCH": "amd64"}) - cmd = ["go", "build", "-o", self.osutils.joinpath(artifacts_dir_path, executable_name), source_dir_path] + cmd = [self.runtime_path, "build", "-o", output_path, source_dir_path] p = self.osutils.popen( cmd, diff --git a/aws_lambda_builders/workflows/go_modules/path_resolver.py b/aws_lambda_builders/workflows/go_modules/path_resolver.py new file mode 100644 index 000000000..791c732a8 --- /dev/null +++ b/aws_lambda_builders/workflows/go_modules/path_resolver.py @@ -0,0 +1,25 @@ +""" +Go Path Resolver that looks for the executable by runtime first, before proceeding to 'go' in PATH. +""" +import whichcraft + + +class GoPathResolver(object): + + def __init__(self, runtime, which=None): + self.language = "go" + self.runtime = runtime + self.executables = [self.language] + self.which = which or whichcraft.which + + def _which(self): + for executable in self.executables: + path = self.which(executable) + if path: + return path + raise ValueError("Path resolution for runtime: {} of language: " + "{} was not successful".format(self.runtime, self.language)) + + @property + def exec_path(self): + return self._which() diff --git a/aws_lambda_builders/workflows/go_modules/utils.py b/aws_lambda_builders/workflows/go_modules/utils.py index 9c637cbcf..2efc79aca 100644 --- a/aws_lambda_builders/workflows/go_modules/utils.py +++ b/aws_lambda_builders/workflows/go_modules/utils.py @@ -13,14 +13,11 @@ class OSUtils(object): """ @property def environ(self): - return os.environ + return os.environ.copy() def joinpath(self, *args): return os.path.join(*args) - def basename(self, *args): - return os.path.basename(*args) - def popen(self, command, stdout=None, stderr=None, env=None, cwd=None): p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd) return p diff --git a/aws_lambda_builders/workflows/go_modules/validator.py b/aws_lambda_builders/workflows/go_modules/validator.py new file mode 100644 index 000000000..a9500c16e --- /dev/null +++ b/aws_lambda_builders/workflows/go_modules/validator.py @@ -0,0 +1,57 @@ +""" +Go Runtime Validation +""" + +import logging +import os +import subprocess + +from aws_lambda_builders.exceptions import MisMatchRuntimeError + +LOG = logging.getLogger(__name__) + + +class GoRuntimeValidator(object): + SUPPORTED_RUNTIMES = { + "go1.x" + } + + def __init__(self, runtime, runtime_path): + self.language = "go" + self.runtime = runtime + self.runtime_path = runtime_path + + def has_runtime(self): + """ + Checks if the runtime is supported. + :param string runtime: Runtime to check + :return bool: True, if the runtime is supported. + """ + return self.runtime in self.SUPPORTED_RUNTIMES + + def validate_runtime(self): + """ + Checks if the language supplied matches the required lambda runtime + :param string runtime_path: runtime to check eg: /usr/bin/go + :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime + """ + if not self.has_runtime(): + LOG.warning("'%s' runtime is not " + "a supported runtime", self.runtime_path) + return + + expected_major_version = self.runtime.replace(self.language, "").split('.')[0] + + p = subprocess.Popen([self.runtime_path, "version"], + cwd=os.getcwd(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, _ = p.communicate() + + mismatched = p.returncode != 0 \ + or len(out.split()) < 3 \ + or out.split()[2].replace(self.language, "").split('.')[0] != expected_major_version + if mismatched: + raise MisMatchRuntimeError(language=self.language, + found_runtime=self.runtime_path, + required_runtime=self.runtime, + runtime_path=self.runtime_path) diff --git a/aws_lambda_builders/workflows/go_modules/workflow.py b/aws_lambda_builders/workflows/go_modules/workflow.py index 67fd03811..3cd096ba7 100644 --- a/aws_lambda_builders/workflows/go_modules/workflow.py +++ b/aws_lambda_builders/workflows/go_modules/workflow.py @@ -2,10 +2,11 @@ Go Modules Workflow """ from aws_lambda_builders.workflow import BaseWorkflow, Capability -from aws_lambda_builders.actions import CopySourceAction from .actions import GoModulesBuildAction from .builder import GoModulesBuilder +from .path_resolver import GoPathResolver +from .validator import GoRuntimeValidator from .utils import OSUtils @@ -37,9 +38,18 @@ def __init__(self, if osutils is None: osutils = OSUtils() - executable_name = osutils.basename(source_dir) - builder = GoModulesBuilder(osutils) + options = kwargs.get("options") or {} + handler = options.get("handler", None) + + output_path = osutils.joinpath(artifacts_dir, handler) + + builder = GoModulesBuilder(osutils, runtime_path=self.get_executable()) self.actions = [ - GoModulesBuildAction(source_dir, artifacts_dir, executable_name, builder), - CopySourceAction(source_dir, artifacts_dir, only=[executable_name]), + GoModulesBuildAction(source_dir, output_path, builder), ] + + def get_executable(self): + return GoPathResolver(runtime=self.runtime).exec_path + + def get_validator(self): + return GoRuntimeValidator(runtime=self.runtime, runtime_path=self.get_executable()) diff --git a/requirements/base.txt b/requirements/base.txt index 0150babf3..e393e5437 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1 +1,2 @@ six~=1.11 +whichcraft~=0.5.2 diff --git a/tests/functional/workflows/go_modules/test_go_utils.py b/tests/functional/workflows/go_modules/test_go_utils.py index 1581f949e..1264af818 100644 --- a/tests/functional/workflows/go_modules/test_go_utils.py +++ b/tests/functional/workflows/go_modules/test_go_utils.py @@ -19,10 +19,6 @@ def test_joinpath_joins_path_components(self): result = self.osutils.joinpath('a', 'b', 'c') self.assertEqual(result, os.path.join('a', 'b', 'c')) - def test_basename_returns_path_basename(self): - result = self.osutils.basename(os.path.dirname(__file__)) - self.assertEqual(result, 'go_modules') - def test_popen_runs_a_process_and_returns_outcome(self): cwd_py = os.path.join(os.path.dirname(__file__), '..', '..', 'testdata', 'cwd.py') p = self.osutils.popen([sys.executable, cwd_py], diff --git a/tests/integration/workflows/go_modules/test_go.py b/tests/integration/workflows/go_modules/test_go.py index 7d3af7b1a..1e138d4a6 100644 --- a/tests/integration/workflows/go_modules/test_go.py +++ b/tests/integration/workflows/go_modules/test_go.py @@ -21,6 +21,7 @@ def setUp(self): self.builder = LambdaBuilder(language="go", dependency_manager="modules", application_framework=None) + self.runtime = "go1.x" def tearDown(self): shutil.rmtree(self.artifacts_dir) @@ -29,8 +30,10 @@ def tearDown(self): def test_builds_project_without_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps") self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "go.mod")) - expected_files = {"no-deps"} + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"handler": "no-deps-main"}) + expected_files = {"no-deps-main"} output_files = set(os.listdir(self.artifacts_dir)) print(output_files) self.assertEquals(expected_files, output_files) @@ -38,8 +41,10 @@ def test_builds_project_without_dependencies(self): def test_builds_project_with_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps") self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "go.mod")) - expected_files = {"with-deps"} + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"handler": "with-deps-main"}) + expected_files = {"with-deps-main"} output_files = set(os.listdir(self.artifacts_dir)) self.assertEquals(expected_files, output_files) @@ -47,6 +52,8 @@ def test_fails_if_modules_cannot_resolve_dependencies(self): source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps") with self.assertRaises(WorkflowFailedError) as ctx: self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, - os.path.join(source_dir, "go.mod")) + os.path.join(source_dir, "go.mod"), + runtime=self.runtime, + options={"handler": "failed"}) self.assertIn("GoModulesBuilder:Build - Builder Failed: ", str(ctx.exception)) diff --git a/tests/unit/workflows/go_modules/test_actions.py b/tests/unit/workflows/go_modules/test_actions.py index cbb7c2bd9..b126442c6 100644 --- a/tests/unit/workflows/go_modules/test_actions.py +++ b/tests/unit/workflows/go_modules/test_actions.py @@ -10,15 +10,15 @@ class TestGoModulesBuildAction(TestCase): @patch("aws_lambda_builders.workflows.go_modules.builder.GoModulesBuilder") def test_runs_bundle_install(self, BuilderMock): builder = BuilderMock.return_value - action = GoModulesBuildAction("source_dir", "artifacts_dir", "executable_name", builder) + action = GoModulesBuildAction("source_dir", "output_path", builder) action.execute() - builder.build.assert_called_with("source_dir", "artifacts_dir", "executable_name") + builder.build.assert_called_with("source_dir", "output_path") @patch("aws_lambda_builders.workflows.go_modules.builder.GoModulesBuilder") def test_raises_action_failed_on_failure(self, BuilderMock): builder = BuilderMock.return_value builder.build.side_effect = BuilderError(message="Fail") - action = GoModulesBuildAction("source_dir", "artifacts_dir", "executable_name", builder) + action = GoModulesBuildAction("source_dir", "output_path", builder) with self.assertRaises(ActionFailedError) as raised: action.execute() self.assertEqual(raised.exception.args[0], "Builder Failed: Fail") diff --git a/tests/unit/workflows/go_modules/test_builder.py b/tests/unit/workflows/go_modules/test_builder.py index 6587cc26f..ee608c69c 100644 --- a/tests/unit/workflows/go_modules/test_builder.py +++ b/tests/unit/workflows/go_modules/test_builder.py @@ -14,7 +14,7 @@ def communicate(self): return self.out, self.err -class TestSubprocessBundler(TestCase): +class TestGoBuilder(TestCase): @patch("aws_lambda_builders.workflows.go_modules.utils.OSUtils") def setUp(self, OSUtilMock): @@ -22,15 +22,14 @@ def setUp(self, OSUtilMock): self.osutils.pipe = 'PIPE' self.popen = FakePopen() self.osutils.popen.side_effect = [self.popen] - self.under_test = GoModulesBuilder(self.osutils) + self.under_test = GoModulesBuilder(self.osutils, "go") def test_run_executes_bundler_on_nixes(self): self.osutils.is_windows.side_effect = [False] - self.under_test = GoModulesBuilder(self.osutils) - self.under_test.build("source_dir", "artifacts_dir", "executable_name") + self.under_test = GoModulesBuilder(self.osutils, "go") + self.under_test.build("source_dir", "output_path") self.osutils.popen.assert_called_with( - ["go", "build", "-o", - self.osutils.joinpath("artifacts_dir", "executable_name"), "source_dir"], + ["go", "build", "-o", "output_path", "source_dir"], cwd="source_dir", env={'GOOS': 'linux', 'GOARCH': 'amd64'}, stderr='PIPE', @@ -39,12 +38,12 @@ def test_run_executes_bundler_on_nixes(self): def test_returns_popen_out_decoded_if_retcode_is_0(self): self.popen.out = b'some encoded text\n\n' - result = self.under_test.build("source_dir", "artifacts_dir", "executable_name") + result = self.under_test.build("source_dir", "output_path") self.assertEqual(result, 'some encoded text') def test_raises_BuilderError_with_err_text_if_retcode_is_not_0(self): self.popen.returncode = 1 self.popen.err = b'some error text\n\n' with self.assertRaises(BuilderError) as raised: - self.under_test.build("source_dir", "artifacts_dir", "executable_name") + self.under_test.build("source_dir", "output_path") self.assertEqual(raised.exception.args[0], "Builder Failed: some error text") diff --git a/tests/unit/workflows/go_modules/test_path_resolver.py b/tests/unit/workflows/go_modules/test_path_resolver.py new file mode 100644 index 000000000..21c793f76 --- /dev/null +++ b/tests/unit/workflows/go_modules/test_path_resolver.py @@ -0,0 +1,28 @@ +import os +from unittest import TestCase + +import mock + +from aws_lambda_builders.workflows.go_modules.path_resolver import GoPathResolver + + +class TestPythonPathResolver(TestCase): + + def setUp(self): + self.path_resolver = GoPathResolver(runtime="go1.x") + + def test_inits(self): + self.assertEquals(self.path_resolver.language, "go") + self.assertEquals(self.path_resolver.runtime, "go1.x") + self.assertEquals(self.path_resolver.executables, + [self.path_resolver.language]) + + def test_which_fails(self): + path_resolver = GoPathResolver(runtime="go1.x", which=lambda x: None) + with self.assertRaises(ValueError): + path_resolver._which() + + def test_which_success_immediate(self): + with mock.patch.object(self.path_resolver, '_which') as which_mock: + which_mock.return_value = os.getcwd() + self.assertEquals(self.path_resolver.exec_path, os.getcwd()) diff --git a/tests/unit/workflows/go_modules/test_validator.py b/tests/unit/workflows/go_modules/test_validator.py new file mode 100644 index 000000000..fd18a354a --- /dev/null +++ b/tests/unit/workflows/go_modules/test_validator.py @@ -0,0 +1,48 @@ +from unittest import TestCase + +import mock +from parameterized import parameterized + +from aws_lambda_builders.exceptions import MisMatchRuntimeError +from aws_lambda_builders.workflows.go_modules.validator import GoRuntimeValidator + + +class MockSubProcess(object): + + def __init__(self, returncode, out=b"", err=b""): + self.returncode = returncode + self.out = out + self.err = err + + def communicate(self): + return (self.out, self.err) + + +class TestGoRuntimeValidator(TestCase): + + def setUp(self): + self.validator = GoRuntimeValidator(runtime="go1.x", runtime_path="/usr/bin/go") + + @parameterized.expand([ + ("go1.x", "/usr/bin/go"), + ]) + def test_supported_runtimes(self, runtime, runtime_path): + validator = GoRuntimeValidator(runtime=runtime, runtime_path=runtime_path) + self.assertTrue(validator.has_runtime()) + + def test_runtime_validate_unsupported_language_fail_open(self): + validator = GoRuntimeValidator(runtime='go2.x', runtime_path='/usr/bin/go2') + validator.validate_runtime() + + def test_runtime_validate_supported_version_runtime(self): + with mock.patch('subprocess.Popen') as mock_subprocess: + mock_subprocess.return_value = MockSubProcess(0, out='go version go1.11.2 test') + self.validator.validate_runtime() + self.assertTrue(mock_subprocess.call_count, 1) + + def test_runtime_validate_mismatch_version_runtime(self): + with mock.patch('subprocess.Popen') as mock_subprocess: + mock_subprocess.return_value = MockSubProcess(1) + with self.assertRaises(MisMatchRuntimeError): + self.validator.validate_runtime() + self.assertTrue(mock_subprocess.call_count, 1) diff --git a/tests/unit/workflows/go_modules/test_workflow.py b/tests/unit/workflows/go_modules/test_workflow.py index 603386954..f27fafc05 100644 --- a/tests/unit/workflows/go_modules/test_workflow.py +++ b/tests/unit/workflows/go_modules/test_workflow.py @@ -1,6 +1,5 @@ from unittest import TestCase -from aws_lambda_builders.actions import CopySourceAction from aws_lambda_builders.workflows.go_modules.workflow import GoModulesWorkflow from aws_lambda_builders.workflows.go_modules.actions import GoModulesBuildAction @@ -12,7 +11,9 @@ class TestGoModulesWorkflow(TestCase): """ def test_workflow_sets_up_builder_actions(self): - workflow = GoModulesWorkflow("source", "artifacts", "scratch_dir", "manifest") - self.assertEqual(len(workflow.actions), 2) + workflow = GoModulesWorkflow( + "source", "artifacts", "scratch_dir", "manifest", + runtime="go1.x", + options={"handler": "main"}) + self.assertEqual(len(workflow.actions), 1) self.assertIsInstance(workflow.actions[0], GoModulesBuildAction) - self.assertIsInstance(workflow.actions[1], CopySourceAction) From 9ecff97fb872c93faf425db84c407aedf33f5224 Mon Sep 17 00:00:00 2001 From: Volkan Gurel Date: Fri, 18 Jan 2019 06:41:33 -0800 Subject: [PATCH 4/6] Update go modules path resolvers and validators Update the go models path resolvers and validators to the latest framework merged in #55 --- .../workflows/go_modules/actions.py | 2 +- .../workflows/go_modules/builder.py | 14 ++++++---- .../workflows/go_modules/path_resolver.py | 25 ----------------- .../workflows/go_modules/validator.py | 24 ++++++++++------ .../workflows/go_modules/workflow.py | 10 ++----- .../unit/workflows/go_modules/test_builder.py | 14 +++++++--- .../go_modules/test_path_resolver.py | 28 ------------------- .../workflows/go_modules/test_validator.py | 22 +++++++-------- 8 files changed, 49 insertions(+), 90 deletions(-) delete mode 100644 aws_lambda_builders/workflows/go_modules/path_resolver.py delete mode 100644 tests/unit/workflows/go_modules/test_path_resolver.py diff --git a/aws_lambda_builders/workflows/go_modules/actions.py b/aws_lambda_builders/workflows/go_modules/actions.py index b867ddb95..39ddedc6c 100644 --- a/aws_lambda_builders/workflows/go_modules/actions.py +++ b/aws_lambda_builders/workflows/go_modules/actions.py @@ -10,7 +10,7 @@ class GoModulesBuildAction(BaseAction): NAME = "Build" DESCRIPTION = "Building Go package with Go Modules" - PURPOSE = Purpose.RESOLVE_DEPENDENCIES + PURPOSE = Purpose.COMPILE_SOURCE def __init__(self, source_dir, output_path, builder): self.source_dir = source_dir diff --git a/aws_lambda_builders/workflows/go_modules/builder.py b/aws_lambda_builders/workflows/go_modules/builder.py index 07d9cb063..35f649b26 100644 --- a/aws_lambda_builders/workflows/go_modules/builder.py +++ b/aws_lambda_builders/workflows/go_modules/builder.py @@ -15,18 +15,21 @@ def __init__(self, **kwargs): class GoModulesBuilder(object): - def __init__(self, osutils, runtime_path): + + LANGUAGE = "go" + + def __init__(self, osutils, binaries): """Initialize a GoModulesBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the outside OS. - :type runtime_path: str - :param runtime_path: The path to the go runtime. + :type binaries: dict + :param binaries: A dict of language binaries """ self.osutils = osutils - self.runtime_path = runtime_path + self.binaries = binaries def build(self, source_dir_path, output_path): """Builds a go project onto an output path. @@ -40,7 +43,8 @@ def build(self, source_dir_path, output_path): env = {} env.update(self.osutils.environ) env.update({"GOOS": "linux", "GOARCH": "amd64"}) - cmd = [self.runtime_path, "build", "-o", output_path, source_dir_path] + runtime_path = self.binaries[self.LANGUAGE].binary_path + cmd = [runtime_path, "build", "-o", output_path, source_dir_path] p = self.osutils.popen( cmd, diff --git a/aws_lambda_builders/workflows/go_modules/path_resolver.py b/aws_lambda_builders/workflows/go_modules/path_resolver.py deleted file mode 100644 index 791c732a8..000000000 --- a/aws_lambda_builders/workflows/go_modules/path_resolver.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Go Path Resolver that looks for the executable by runtime first, before proceeding to 'go' in PATH. -""" -import whichcraft - - -class GoPathResolver(object): - - def __init__(self, runtime, which=None): - self.language = "go" - self.runtime = runtime - self.executables = [self.language] - self.which = which or whichcraft.which - - def _which(self): - for executable in self.executables: - path = self.which(executable) - if path: - return path - raise ValueError("Path resolution for runtime: {} of language: " - "{} was not successful".format(self.runtime, self.language)) - - @property - def exec_path(self): - return self._which() diff --git a/aws_lambda_builders/workflows/go_modules/validator.py b/aws_lambda_builders/workflows/go_modules/validator.py index a9500c16e..2bef50827 100644 --- a/aws_lambda_builders/workflows/go_modules/validator.py +++ b/aws_lambda_builders/workflows/go_modules/validator.py @@ -16,10 +16,10 @@ class GoRuntimeValidator(object): "go1.x" } - def __init__(self, runtime, runtime_path): + def __init__(self, runtime): self.language = "go" self.runtime = runtime - self.runtime_path = runtime_path + self._valid_runtime_path = None def has_runtime(self): """ @@ -29,7 +29,7 @@ def has_runtime(self): """ return self.runtime in self.SUPPORTED_RUNTIMES - def validate_runtime(self): + def validate(self, runtime_path): """ Checks if the language supplied matches the required lambda runtime :param string runtime_path: runtime to check eg: /usr/bin/go @@ -37,21 +37,27 @@ def validate_runtime(self): """ if not self.has_runtime(): LOG.warning("'%s' runtime is not " - "a supported runtime", self.runtime_path) - return + "a supported runtime", self.runtime) + return None expected_major_version = self.runtime.replace(self.language, "").split('.')[0] - p = subprocess.Popen([self.runtime_path, "version"], + p = subprocess.Popen([runtime_path, "version"], cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _ = p.communicate() mismatched = p.returncode != 0 \ or len(out.split()) < 3 \ - or out.split()[2].replace(self.language, "").split('.')[0] != expected_major_version + or out.split()[2].decode().replace(self.language, "").split('.')[0] != expected_major_version if mismatched: raise MisMatchRuntimeError(language=self.language, - found_runtime=self.runtime_path, required_runtime=self.runtime, - runtime_path=self.runtime_path) + runtime_path=runtime_path) + else: + self._valid_runtime_path = runtime_path + return self._valid_runtime_path + + @property + def validated_runtime_path(self): + return self._valid_runtime_path if self._valid_runtime_path is not None else None diff --git a/aws_lambda_builders/workflows/go_modules/workflow.py b/aws_lambda_builders/workflows/go_modules/workflow.py index 3cd096ba7..b19666292 100644 --- a/aws_lambda_builders/workflows/go_modules/workflow.py +++ b/aws_lambda_builders/workflows/go_modules/workflow.py @@ -5,7 +5,6 @@ from .actions import GoModulesBuildAction from .builder import GoModulesBuilder -from .path_resolver import GoPathResolver from .validator import GoRuntimeValidator from .utils import OSUtils @@ -43,13 +42,10 @@ def __init__(self, output_path = osutils.joinpath(artifacts_dir, handler) - builder = GoModulesBuilder(osutils, runtime_path=self.get_executable()) + builder = GoModulesBuilder(osutils, binaries=self.binaries) self.actions = [ GoModulesBuildAction(source_dir, output_path, builder), ] - def get_executable(self): - return GoPathResolver(runtime=self.runtime).exec_path - - def get_validator(self): - return GoRuntimeValidator(runtime=self.runtime, runtime_path=self.get_executable()) + def get_validators(self): + return [GoRuntimeValidator(runtime=self.runtime)] diff --git a/tests/unit/workflows/go_modules/test_builder.py b/tests/unit/workflows/go_modules/test_builder.py index ee608c69c..9e69db981 100644 --- a/tests/unit/workflows/go_modules/test_builder.py +++ b/tests/unit/workflows/go_modules/test_builder.py @@ -1,6 +1,8 @@ from unittest import TestCase -from mock import patch +from mock import patch, Mock + +from aws_lambda_builders.binary_path import BinaryPath from aws_lambda_builders.workflows.go_modules.builder import GoModulesBuilder, BuilderError @@ -22,14 +24,18 @@ def setUp(self, OSUtilMock): self.osutils.pipe = 'PIPE' self.popen = FakePopen() self.osutils.popen.side_effect = [self.popen] - self.under_test = GoModulesBuilder(self.osutils, "go") + self.binaries = { + "go": BinaryPath(resolver=Mock(), validator=Mock(), + binary="go", binary_path="/path/to/go") + } + self.under_test = GoModulesBuilder(self.osutils, self.binaries) def test_run_executes_bundler_on_nixes(self): self.osutils.is_windows.side_effect = [False] - self.under_test = GoModulesBuilder(self.osutils, "go") + self.under_test = GoModulesBuilder(self.osutils, self.binaries) self.under_test.build("source_dir", "output_path") self.osutils.popen.assert_called_with( - ["go", "build", "-o", "output_path", "source_dir"], + ["/path/to/go", "build", "-o", "output_path", "source_dir"], cwd="source_dir", env={'GOOS': 'linux', 'GOARCH': 'amd64'}, stderr='PIPE', diff --git a/tests/unit/workflows/go_modules/test_path_resolver.py b/tests/unit/workflows/go_modules/test_path_resolver.py deleted file mode 100644 index 21c793f76..000000000 --- a/tests/unit/workflows/go_modules/test_path_resolver.py +++ /dev/null @@ -1,28 +0,0 @@ -import os -from unittest import TestCase - -import mock - -from aws_lambda_builders.workflows.go_modules.path_resolver import GoPathResolver - - -class TestPythonPathResolver(TestCase): - - def setUp(self): - self.path_resolver = GoPathResolver(runtime="go1.x") - - def test_inits(self): - self.assertEquals(self.path_resolver.language, "go") - self.assertEquals(self.path_resolver.runtime, "go1.x") - self.assertEquals(self.path_resolver.executables, - [self.path_resolver.language]) - - def test_which_fails(self): - path_resolver = GoPathResolver(runtime="go1.x", which=lambda x: None) - with self.assertRaises(ValueError): - path_resolver._which() - - def test_which_success_immediate(self): - with mock.patch.object(self.path_resolver, '_which') as which_mock: - which_mock.return_value = os.getcwd() - self.assertEquals(self.path_resolver.exec_path, os.getcwd()) diff --git a/tests/unit/workflows/go_modules/test_validator.py b/tests/unit/workflows/go_modules/test_validator.py index fd18a354a..b275e107e 100644 --- a/tests/unit/workflows/go_modules/test_validator.py +++ b/tests/unit/workflows/go_modules/test_validator.py @@ -21,28 +21,28 @@ def communicate(self): class TestGoRuntimeValidator(TestCase): def setUp(self): - self.validator = GoRuntimeValidator(runtime="go1.x", runtime_path="/usr/bin/go") + self.validator = GoRuntimeValidator(runtime="go1.x") @parameterized.expand([ - ("go1.x", "/usr/bin/go"), + "go1.x", ]) - def test_supported_runtimes(self, runtime, runtime_path): - validator = GoRuntimeValidator(runtime=runtime, runtime_path=runtime_path) + def test_supported_runtimes(self, runtime): + validator = GoRuntimeValidator(runtime=runtime) self.assertTrue(validator.has_runtime()) def test_runtime_validate_unsupported_language_fail_open(self): - validator = GoRuntimeValidator(runtime='go2.x', runtime_path='/usr/bin/go2') - validator.validate_runtime() + validator = GoRuntimeValidator(runtime="go2.x") + validator.validate(runtime_path="/usr/bin/go2") def test_runtime_validate_supported_version_runtime(self): - with mock.patch('subprocess.Popen') as mock_subprocess: - mock_subprocess.return_value = MockSubProcess(0, out='go version go1.11.2 test') - self.validator.validate_runtime() + with mock.patch("subprocess.Popen") as mock_subprocess: + mock_subprocess.return_value = MockSubProcess(0, out=b"go version go1.11.2 test") + self.validator.validate(runtime_path="/usr/bin/go") self.assertTrue(mock_subprocess.call_count, 1) def test_runtime_validate_mismatch_version_runtime(self): - with mock.patch('subprocess.Popen') as mock_subprocess: + with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(1) with self.assertRaises(MisMatchRuntimeError): - self.validator.validate_runtime() + self.validator.validate(runtime_path="/usr/bin/go") self.assertTrue(mock_subprocess.call_count, 1) From e3938e99033c552df20f7f063ff6873de6cce192 Mon Sep 17 00:00:00 2001 From: Volkan Gurel Date: Fri, 18 Jan 2019 16:18:01 -0800 Subject: [PATCH 5/6] Address PR feedback --- .../workflows/go_modules/validator.py | 29 +++++++++++-------- requirements/base.txt | 1 - .../workflows/go_modules/test_validator.py | 16 +++++++++- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/aws_lambda_builders/workflows/go_modules/validator.py b/aws_lambda_builders/workflows/go_modules/validator.py index 2bef50827..5b4110524 100644 --- a/aws_lambda_builders/workflows/go_modules/validator.py +++ b/aws_lambda_builders/workflows/go_modules/validator.py @@ -12,12 +12,13 @@ class GoRuntimeValidator(object): + + LANGUAGE = "go" SUPPORTED_RUNTIMES = { "go1.x" } def __init__(self, runtime): - self.language = "go" self.runtime = runtime self._valid_runtime_path = None @@ -40,23 +41,27 @@ def validate(self, runtime_path): "a supported runtime", self.runtime) return None - expected_major_version = self.runtime.replace(self.language, "").split('.')[0] + expected_major_version = int(self.runtime.replace(self.LANGUAGE, "").split('.')[0]) + min_expected_minor_version = 11 if expected_major_version == 1 else 0 p = subprocess.Popen([runtime_path, "version"], cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, _ = p.communicate() - mismatched = p.returncode != 0 \ - or len(out.split()) < 3 \ - or out.split()[2].decode().replace(self.language, "").split('.')[0] != expected_major_version - if mismatched: - raise MisMatchRuntimeError(language=self.language, - required_runtime=self.runtime, - runtime_path=runtime_path) - else: - self._valid_runtime_path = runtime_path - return self._valid_runtime_path + if p.returncode == 0: + out_parts = out.decode().split() + if len(out_parts) >= 3: + version_parts = [int(x) for x in out_parts[2].replace(self.LANGUAGE, "").split('.')] + if len(version_parts) == 3: + if version_parts[0] == expected_major_version and version_parts[1] >= min_expected_minor_version: + self._valid_runtime_path = runtime_path + return self._valid_runtime_path + + # otherwise, raise mismatch exception + raise MisMatchRuntimeError(language=self.LANGUAGE, + required_runtime=self.runtime, + runtime_path=runtime_path) @property def validated_runtime_path(self): diff --git a/requirements/base.txt b/requirements/base.txt index e393e5437..0150babf3 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,2 +1 @@ six~=1.11 -whichcraft~=0.5.2 diff --git a/tests/unit/workflows/go_modules/test_validator.py b/tests/unit/workflows/go_modules/test_validator.py index b275e107e..9e3931de7 100644 --- a/tests/unit/workflows/go_modules/test_validator.py +++ b/tests/unit/workflows/go_modules/test_validator.py @@ -40,9 +40,23 @@ def test_runtime_validate_supported_version_runtime(self): self.validator.validate(runtime_path="/usr/bin/go") self.assertTrue(mock_subprocess.call_count, 1) - def test_runtime_validate_mismatch_version_runtime(self): + def test_runtime_validate_mismatch_nonzero_exit(self): with mock.patch("subprocess.Popen") as mock_subprocess: mock_subprocess.return_value = MockSubProcess(1) with self.assertRaises(MisMatchRuntimeError): self.validator.validate(runtime_path="/usr/bin/go") self.assertTrue(mock_subprocess.call_count, 1) + + def test_runtime_validate_mismatch_invalid_version(self): + with mock.patch("subprocess.Popen") as mock_subprocess: + mock_subprocess.return_value = MockSubProcess(0, out=b"go version") + with self.assertRaises(MisMatchRuntimeError): + self.validator.validate(runtime_path="/usr/bin/go") + self.assertTrue(mock_subprocess.call_count, 1) + + def test_runtime_validate_mismatch_minor_version(self): + with mock.patch("subprocess.Popen") as mock_subprocess: + mock_subprocess.return_value = MockSubProcess(0, out=b"go version go1.10.2 test") + with self.assertRaises(MisMatchRuntimeError): + self.validator.validate(runtime_path="/usr/bin/go") + self.assertTrue(mock_subprocess.call_count, 1) From dae82d69ef063b4f6e4951a0c78cef4a965cfa3d Mon Sep 17 00:00:00 2001 From: Volkan Gurel Date: Wed, 23 Jan 2019 09:18:53 -0800 Subject: [PATCH 6/6] Address PR feedback --- aws_lambda_builders/workflows/go_modules/validator.py | 2 +- aws_lambda_builders/workflows/python_pip/validator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_lambda_builders/workflows/go_modules/validator.py b/aws_lambda_builders/workflows/go_modules/validator.py index 5b4110524..e44779efc 100644 --- a/aws_lambda_builders/workflows/go_modules/validator.py +++ b/aws_lambda_builders/workflows/go_modules/validator.py @@ -65,4 +65,4 @@ def validate(self, runtime_path): @property def validated_runtime_path(self): - return self._valid_runtime_path if self._valid_runtime_path is not None else None + return self._valid_runtime_path diff --git a/aws_lambda_builders/workflows/python_pip/validator.py b/aws_lambda_builders/workflows/python_pip/validator.py index e5d44f691..22e50a2d3 100644 --- a/aws_lambda_builders/workflows/python_pip/validator.py +++ b/aws_lambda_builders/workflows/python_pip/validator.py @@ -70,4 +70,4 @@ def _validate_python_cmd(self, runtime_path): @property def validated_runtime_path(self): - return self._valid_runtime_path if self._valid_runtime_path is not None else None + return self._valid_runtime_path