diff --git a/.appveyor.yml b/.appveyor.yml index 8e98574cc..4b252170f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -23,7 +23,7 @@ install: - "%PYTHON%\\python.exe -m pip install -e ." - "set PATH=C:\\Ruby25-x64\\bin;%PATH%" - "gem --version" -- "gem install bundler -v 1.17.3 --no-ri --no-rdoc" +- "gem install bundler -v 1.17.3" - "bundler --version" - "echo %PATH%" diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index 1dc20590b..8cf24ecf1 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -2,4 +2,4 @@ AWS Lambda Builder Library """ __version__ = '0.0.5' -RPC_PROTOCOL_VERSION = "0.1" +RPC_PROTOCOL_VERSION = "0.2" diff --git a/aws_lambda_builders/__main__.py b/aws_lambda_builders/__main__.py index 142f4eb1e..3331478c7 100644 --- a/aws_lambda_builders/__main__.py +++ b/aws_lambda_builders/__main__.py @@ -10,10 +10,11 @@ import json import os import logging +import re from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowNotFoundError, WorkflowUnknownError, WorkflowFailedError - +from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version log_level = int(os.environ.get("LAMBDA_BUILDERS_LOG_LEVEL", logging.INFO)) @@ -24,6 +25,8 @@ LOG = logging.getLogger(__name__) +VERSION_REGEX = re.compile("^([0-9])+.([0-9]+)$") + def _success_response(request_id, artifacts_dir): return json.dumps({ @@ -46,6 +49,31 @@ def _error_response(request_id, http_status_code, message): }) +def _parse_version(version_string): + + if VERSION_REGEX.match(version_string): + return float(version_string) + else: + ex = "Protocol Version does not match : {}".format(VERSION_REGEX.pattern) + LOG.debug(ex) + raise ValueError(ex) + + +def version_compatibility_check(version): + # The following check is between current protocol version vs version of the protocol + # with which aws-lambda-builders is called. + # Example: + # 0.2 < 0.2 comparison will fail, don't throw a value Error saying incompatible version. + # 0.2 < 0.3 comparison will pass, throwing a ValueError + # 0.2 < 0.1 comparison will fail, don't throw a value Error saying incompatible version + + if _parse_version(lambda_builders_protocol_version) < version: + ex = "Incompatible Protocol Version : {}, " \ + "Current Protocol Version: {}".format(version, lambda_builders_protocol_version) + LOG.error(ex) + raise ValueError(ex) + + def _write_response(response, exit_code): sys.stdout.write(response) sys.stdout.flush() # Make sure it is written @@ -77,11 +105,20 @@ def main(): # pylint: disable=too-many-statements response = _error_response(request_id, -32601, "Method unavailable") return _write_response(response, 1) + try: + protocol_version = _parse_version(params.get("__protocol_version")) + version_compatibility_check(protocol_version) + + except ValueError: + response = _error_response(request_id, 505, "Unsupported Protocol Version") + return _write_response(response, 1) + capabilities = params["capability"] supported_workflows = params.get("supported_workflows") exit_code = 0 response = None + try: builder = LambdaBuilder(language=capabilities["language"], dependency_manager=capabilities["dependency_manager"], @@ -93,6 +130,7 @@ def main(): # pylint: disable=too-many-statements params["artifacts_dir"], params["scratch_dir"], params["manifest_path"], + executable_search_paths=params['executable_search_paths'], runtime=params["runtime"], optimizations=params["optimizations"], options=params["options"]) diff --git a/aws_lambda_builders/builder.py b/aws_lambda_builders/builder.py index 2de4c53ba..1012a5d02 100644 --- a/aws_lambda_builders/builder.py +++ b/aws_lambda_builders/builder.py @@ -56,7 +56,7 @@ def __init__(self, language, dependency_manager, application_framework, supporte LOG.debug("Found workflow '%s' to support capabilities '%s'", self.selected_workflow_cls.NAME, self.capability) def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path, - runtime=None, optimizations=None, options=None): + runtime=None, optimizations=None, options=None, executable_search_paths=None): """ Actually build the code by running workflows @@ -89,6 +89,10 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path, :type options: dict :param options: Optional dictionary of options ot pass to build action. **Not supported**. + + :type executable_search_paths: list + :param executable_search_paths: + Additional list of paths to search for executables required by the workflow. """ if not os.path.exists(scratch_dir): @@ -100,7 +104,8 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path, manifest_path, runtime=runtime, optimizations=optimizations, - options=options) + options=options, + executable_search_paths=executable_search_paths) return workflow.run() diff --git a/aws_lambda_builders/path_resolver.py b/aws_lambda_builders/path_resolver.py index db7978f2f..3d05e135c 100644 --- a/aws_lambda_builders/path_resolver.py +++ b/aws_lambda_builders/path_resolver.py @@ -7,15 +7,16 @@ class PathResolver(object): - def __init__(self, binary, runtime): + def __init__(self, binary, runtime, executable_search_paths=None): self.binary = binary self.runtime = runtime self.executables = [self.runtime, self.binary] + self.executable_search_paths = executable_search_paths def _which(self): exec_paths = [] for executable in [executable for executable in self.executables if executable is not None]: - paths = which(executable) + paths = which(executable, executable_search_paths=self.executable_search_paths) exec_paths.extend(paths) if not exec_paths: diff --git a/aws_lambda_builders/utils.py b/aws_lambda_builders/utils.py index b33cd8964..d5b2ec9aa 100644 --- a/aws_lambda_builders/utils.py +++ b/aws_lambda_builders/utils.py @@ -68,15 +68,27 @@ def copytree(source, destination, ignore=None): # Copyright 2019 by the Python Software Foundation -def which(cmd, mode=os.F_OK | os.X_OK, path=None): # pragma: no cover - """Given a command, mode, and a PATH string, return the paths which - conforms to the given mode on the PATH, or None if there is no such - file. - `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result - of os.environ.get("PATH"), or can be overridden with a custom search - path. +def which(cmd, mode=os.F_OK | os.X_OK, executable_search_paths=None): # pragma: no cover + """Given a command, mode, and executable search paths list, return the paths which + conforms to the given mode on the PATH with the prepended additional search paths, + or None if there is no such file. + `mode` defaults to os.F_OK | os.X_OK. the default search `path` defaults + to the result of os.environ.get("PATH") Note: This function was backported from the Python 3 source code. + + :type cmd: str + :param cmd: + Executable to be looked up in PATH. + + :type mode: str + :param mode: + Modes of access for the executable. + + :type executable_search_paths: list + :param executable_search_paths: + List of paths to look for `cmd` in preference order. """ + # Check that a given file can be accessed with the correct mode. # Additionally check that `file` is not a directory, as on Windows # directories pass the os.access check. @@ -93,13 +105,16 @@ def _access_check(fn, mode): return None - if path is None: - path = os.environ.get("PATH", os.defpath) + path = os.environ.get("PATH", os.defpath) + if not path: return None path = path.split(os.pathsep) + if executable_search_paths: + path = executable_search_paths + path + if sys.platform == "win32": # The current directory takes precedence on Windows. if os.curdir not in path: diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index fafdcf977..8a6032602 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -117,6 +117,7 @@ def __init__(self, scratch_dir, manifest_path, runtime=None, + executable_search_paths=None, optimizations=None, options=None): """ @@ -152,6 +153,10 @@ def __init__(self, :type options: dict :param options: Optional dictionary of options ot pass to build action. **Not supported**. + + :type executable_search_paths: list + :param executable_search_paths: + Optional, Additional list of paths to search for executables required by the workflow. """ self.source_dir = source_dir @@ -161,6 +166,7 @@ def __init__(self, self.runtime = runtime self.optimizations = optimizations self.options = options + self.executable_search_paths = executable_search_paths # Actions are registered by the subclasses as they seem fit self.actions = [] @@ -181,7 +187,8 @@ def get_resolvers(self): """ Non specialized path resolver that just returns the list of executable for the runtime on the path. """ - return [PathResolver(runtime=self.runtime, binary=self.CAPABILITY.language)] + return [PathResolver(runtime=self.runtime, binary=self.CAPABILITY.language, + executable_search_paths=self.executable_search_paths)] def get_validators(self): """ diff --git a/tests/functional/test_builder.py b/tests/functional/test_builder.py index 29f02415e..a5a1b172f 100644 --- a/tests/functional/test_builder.py +++ b/tests/functional/test_builder.py @@ -4,6 +4,11 @@ import shutil import tempfile +try: + import pathlib +except ImportError: + import pathlib2 as pathlib + from unittest import TestCase from aws_lambda_builders.builder import LambdaBuilder @@ -40,16 +45,17 @@ def tearDown(self): # Remove the workflows folder from PYTHONPATH sys.path.remove(self.TEST_WORKFLOWS_FOLDER) - def test_run_hello_workflow(self): + def test_run_hello_workflow_with_exec_paths(self): self.hello_builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, - "/ignored") + "/ignored", + executable_search_paths=[str(pathlib.Path(sys.executable).parent)]) self.assertTrue(os.path.exists(self.expected_filename)) contents = '' with open(self.expected_filename, 'r') as fp: contents = fp.read() - self.assertEquals(contents, self.expected_contents) + self.assertEquals(contents, self.expected_contents) \ No newline at end of file diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index f044f21d8..11dc8a9aa 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -5,10 +5,19 @@ import tempfile import subprocess import copy +import sys from unittest import TestCase from parameterized import parameterized +try: + import pathlib +except ImportError: + import pathlib2 as pathlib + + +from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version + class TestCliWithHelloWorkflow(TestCase): @@ -39,19 +48,21 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.source_dir) shutil.rmtree(self.artifacts_dir) - shutil.rmtree(self.scratch_dir) @parameterized.expand([ - ("request_through_stdin"), - ("request_through_argument") + ("request_through_stdin", lambda_builders_protocol_version), + ("request_through_argument", lambda_builders_protocol_version), + ("request_through_stdin", "0.1"), + ("request_through_argument", "0.1"), ]) - def test_run_hello_workflow(self, flavor): + def test_run_hello_workflow_with_backcompat(self, flavor, protocol_version): request_json = json.dumps({ "jsonschema": "2.0", "id": 1234, "method": "LambdaBuilder.build", "params": { + "__protocol_version": protocol_version, "capability": { "language": self.language, "dependency_manager": self.dependency_manager, @@ -65,6 +76,7 @@ def test_run_hello_workflow(self, flavor): "runtime": "ignored", "optimizations": {}, "options": {}, + "executable_search_paths": [str(pathlib.Path(sys.executable).parent)] } }) @@ -94,4 +106,52 @@ def test_run_hello_workflow(self, flavor): contents = fp.read() self.assertEquals(contents, self.expected_contents) + shutil.rmtree(self.scratch_dir) + + @parameterized.expand([ + ("request_through_stdin"), + ("request_through_argument") + ]) + def test_run_hello_workflow_incompatible(self, flavor): + + request_json = json.dumps({ + "jsonschema": "2.0", + "id": 1234, + "method": "LambdaBuilder.build", + "params": { + "__protocol_version": "2.0", + "capability": { + "language": self.language, + "dependency_manager": self.dependency_manager, + "application_framework": self.application_framework + }, + "supported_workflows": [self.HELLO_WORKFLOW_MODULE], + "source_dir": self.source_dir, + "artifacts_dir": self.artifacts_dir, + "scratch_dir": self.scratch_dir, + "manifest_path": "/ignored", + "runtime": "ignored", + "optimizations": {}, + "options": {}, + "executable_search_paths": [str(pathlib.Path(sys.executable).parent)] + } + }) + + + env = copy.deepcopy(os.environ) + env["PYTHONPATH"] = self.python_path + + stdout_data = None + if flavor == "request_through_stdin": + p = subprocess.Popen([self.command_name], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + stdout_data = p.communicate(input=request_json.encode('utf-8'))[0] + elif flavor == "request_through_argument": + p = subprocess.Popen([self.command_name, request_json], env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + stdout_data = p.communicate()[0] + else: + raise ValueError("Invalid test flavor") + # Validate the response object. It should be error response + response = json.loads(stdout_data) + self.assertIn('error', response) + self.assertEquals(response['error']['code'], 505) diff --git a/tests/functional/test_utils.py b/tests/functional/test_utils.py index 69dcd3390..b82032df0 100644 --- a/tests/functional/test_utils.py +++ b/tests/functional/test_utils.py @@ -40,6 +40,7 @@ def test_must_respect_excludes_list(self): self.assertEquals(set(os.listdir(os.path.join(self.dest, "a"))), {"c"}) self.assertEquals(set(os.listdir(os.path.join(self.dest, "a"))), {"c"}) + def file(*args): path = os.path.join(*args) basedir = os.path.dirname(path) diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 9ef9e8e7e..f5f7051b2 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -78,9 +78,11 @@ def __init__(self, manifest_path, runtime=None, optimizations=None, - options=None): + options=None, + executable_search_paths=None): super(MyWorkflow, self).__init__(source_dir, artifacts_dir, scratch_dir, manifest_path, - runtime=runtime, optimizations=optimizations, options=options) + runtime=runtime, optimizations=optimizations, options=options, + executable_search_paths=executable_search_paths) # Don't load any other workflows. The above class declaration will automatically load the workflow into registry builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework, supported_workflows=[]) @@ -118,10 +120,12 @@ def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework, supported_workflows=[]) builder.build("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", optimizations="optimizations", options="options") + runtime="runtime", optimizations="optimizations", options="options", + executable_search_paths="executable_search_paths") workflow_cls.assert_called_with("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", optimizations="optimizations", options="options") + runtime="runtime", optimizations="optimizations", options="options", + executable_search_paths="executable_search_paths") workflow_instance.run.assert_called_once() os_mock.path.exists.assert_called_once_with("scratch_dir") if scratch_dir_exists: diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index a6a99cef4..08900950a 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -1,7 +1,13 @@ - +import os +import sys from unittest import TestCase from mock import Mock, call +try: + import pathlib +except ImportError: + import pathlib2 as pathlib + from aws_lambda_builders.binary_path import BinaryPath from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.workflow import BaseWorkflow, Capability @@ -89,6 +95,7 @@ class MyWorkflow(BaseWorkflow): def test_must_initialize_variables(self): self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", runtime="runtime", + executable_search_paths=[str(sys.executable)], optimizations={"a": "b"}, options={"c": "d"}) @@ -97,6 +104,7 @@ def test_must_initialize_variables(self): self.assertEquals(self.work.scratch_dir, "scratch_dir") self.assertEquals(self.work.manifest_path, "manifest_path") self.assertEquals(self.work.runtime, "runtime") + self.assertEquals(self.work.executable_search_paths, [str(sys.executable)]) self.assertEquals(self.work.optimizations, {"a": "b"}) self.assertEquals(self.work.options, {"c": "d"}) @@ -113,6 +121,7 @@ class MyWorkflow(BaseWorkflow): def setUp(self): self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", runtime="runtime", + executable_search_paths=[], optimizations={"a": "b"}, options={"c": "d"}) @@ -150,6 +159,7 @@ class MyWorkflow(BaseWorkflow): def setUp(self): self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", runtime="runtime", + executable_search_paths=[], optimizations={"a": "b"}, options={"c": "d"}) @@ -216,6 +226,18 @@ def test_must_raise_if_action_crashed(self): self.assertIn("somevalueerror", str(ctx.exception)) + def test_supply_executable_path(self): + # Run workflow with supplied executable path to search for executables. + action_mock = Mock() + + self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", + runtime="runtime", + executable_search_paths=[str(pathlib.Path(os.getcwd()).parent)], + optimizations={"a": "b"}, + options={"c": "d"}) + self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] + self.work.run() + class TestBaseWorkflow_repr(TestCase): @@ -239,6 +261,7 @@ def setUp(self): self.work = self.MyWorkflow("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", runtime="runtime", + executable_search_paths=[], optimizations={"a": "b"}, options={"c": "d"})