diff --git a/aws_lambda_builders/actions.py b/aws_lambda_builders/actions.py index a7154f66a..7cb65ceac 100644 --- a/aws_lambda_builders/actions.py +++ b/aws_lambda_builders/actions.py @@ -5,8 +5,10 @@ import logging import os import shutil +from pathlib import Path from typing import Set, Iterator, Tuple +from aws_lambda_builders import utils from aws_lambda_builders.utils import copytree LOG = logging.getLogger(__name__) @@ -31,6 +33,9 @@ class Purpose(object): # Action is copying source code COPY_SOURCE = "COPY_SOURCE" + # Action is linking source code + LINK_SOURCE = "LINK_SOURCE" + # Action is copying dependencies COPY_DEPENDENCIES = "COPY_DEPENDENCIES" @@ -111,6 +116,31 @@ def execute(self): copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes)) +class LinkSourceAction(BaseAction): + + NAME = "LinkSource" + + DESCRIPTION = "Linking source code to the target folder" + + PURPOSE = Purpose.LINK_SOURCE + + def __init__(self, source_dir, dest_dir): + self._source_dir = source_dir + self._dest_dir = dest_dir + + def execute(self): + source_files = set(os.listdir(self._source_dir)) + + for source_file in source_files: + source_path = Path(self._source_dir, source_file) + destination_path = Path(self._dest_dir, source_file) + if destination_path.exists(): + os.remove(destination_path) + else: + os.makedirs(destination_path.parent, exist_ok=True) + utils.create_symlink_or_copy(str(source_path), str(destination_path)) + + class CopyDependenciesAction(BaseAction): NAME = "CopyDependencies" @@ -175,10 +205,10 @@ def __init__(self, target_dir): def execute(self): if not os.path.isdir(self.target_dir): - LOG.info("Clean up action: %s does not exist and will be skipped.", str(self.target_dir)) + LOG.debug("Clean up action: %s does not exist and will be skipped.", str(self.target_dir)) return targets = os.listdir(self.target_dir) - LOG.info("Clean up action: folder %s will be cleaned", str(self.target_dir)) + LOG.debug("Clean up action: folder %s will be cleaned", str(self.target_dir)) for name in targets: target_path = os.path.join(self.target_dir, name) diff --git a/aws_lambda_builders/utils.py b/aws_lambda_builders/utils.py index 791b68669..0e5e7447b 100644 --- a/aws_lambda_builders/utils.py +++ b/aws_lambda_builders/utils.py @@ -6,6 +6,7 @@ import sys import os import logging +from pathlib import Path from aws_lambda_builders.architecture import X86_64, ARM64 @@ -182,3 +183,16 @@ def get_goarch(architecture): returns a valid GO Architecture value """ return "arm64" if architecture == ARM64 else "amd64" + + +def create_symlink_or_copy(source: str, destination: str) -> None: + """Tries to create symlink, if it fails it will copy source into destination""" + LOG.debug("Creating symlink; source: %s, destination: %s", source, destination) + try: + os.symlink(Path(source).absolute(), Path(destination).absolute()) + except OSError as ex: + LOG.warning( + "Symlink operation is failed, falling back to copying files", + exc_info=ex if LOG.isEnabledFor(logging.DEBUG) else None, + ) + copytree(source, destination) diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/utils.py b/aws_lambda_builders/workflows/nodejs_npm_esbuild/utils.py index 2d1aadb57..843642e3b 100644 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/utils.py +++ b/aws_lambda_builders/workflows/nodejs_npm_esbuild/utils.py @@ -1,11 +1,12 @@ """ esbuild specific utilities and feature flag """ +from typing import Optional, List EXPERIMENTAL_FLAG_ESBUILD = "experimentalEsbuild" -def is_experimental_esbuild_scope(experimental_flags): +def is_experimental_esbuild_scope(experimental_flags: Optional[List[str]]) -> bool: """ A function which will determine if experimental esbuild scope is active """ diff --git a/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py b/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py index d5cdbb633..70772d97d 100644 --- a/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py @@ -10,9 +10,9 @@ from aws_lambda_builders.actions import ( CopySourceAction, CleanUpAction, - CopyDependenciesAction, MoveDependenciesAction, BaseAction, + LinkSourceAction, ) from aws_lambda_builders.utils import which from .actions import ( @@ -168,13 +168,18 @@ def _accelerate_workflow_actions( actions += [install_action, CleanUpAction(self.dependencies_dir)] if self.combine_dependencies: # Auto dependency layer disabled, first build - actions += [esbuild_with_deps, CopyDependenciesAction(source_dir, scratch_dir, self.dependencies_dir)] + actions += [ + esbuild_with_deps, + MoveDependenciesAction(source_dir, scratch_dir, self.dependencies_dir), + LinkSourceAction(self.dependencies_dir, scratch_dir), + ] else: # Auto dependency layer enabled, first build actions += esbuild_no_deps + [MoveDependenciesAction(source_dir, scratch_dir, self.dependencies_dir)] else: if self.dependencies_dir: - actions.append(CopySourceAction(self.dependencies_dir, scratch_dir)) + actions.append(LinkSourceAction(self.dependencies_dir, scratch_dir)) + if self.combine_dependencies: # Auto dependency layer disabled, subsequent builds actions += [esbuild_with_deps] diff --git a/aws_lambda_builders/workflows/python_pip/utils.py b/aws_lambda_builders/workflows/python_pip/utils.py index 19ee7656d..f5eea5882 100644 --- a/aws_lambda_builders/workflows/python_pip/utils.py +++ b/aws_lambda_builders/workflows/python_pip/utils.py @@ -11,6 +11,9 @@ import tarfile import subprocess import sys +from typing import Optional, List + +EXPERIMENTAL_FLAG_BUILD_PERFORMANCE = "experimentalBuildPerformance" class OSUtils(object): @@ -106,3 +109,10 @@ def pipe(self): def basename(self, path): # type: (str) -> str return os.path.basename(path) + + +def is_experimental_build_improvements_enabled(experimental_flags: Optional[List[str]]) -> bool: + """ + A function which will determine if experimental build improvements is active + """ + return bool(experimental_flags) and EXPERIMENTAL_FLAG_BUILD_PERFORMANCE in experimental_flags diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index 080725097..b0da62340 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -4,11 +4,11 @@ import logging from aws_lambda_builders.workflow import BaseWorkflow, Capability -from aws_lambda_builders.actions import CopySourceAction, CleanUpAction +from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, LinkSourceAction from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator from .actions import PythonPipBuildAction -from .utils import OSUtils +from .utils import OSUtils, is_experimental_build_improvements_enabled LOG = logging.getLogger(__name__) @@ -106,7 +106,10 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim # folder if self.dependencies_dir and self.combine_dependencies: # when copying downloaded dependencies back to artifacts folder, don't exclude anything - self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) + if is_experimental_build_improvements_enabled(self.experimental_flags): + self.actions.append(LinkSourceAction(self.dependencies_dir, artifacts_dir)) + else: + self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir)) self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index ce6bc4aba..526320e98 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -5,23 +5,28 @@ import tempfile from unittest import TestCase, skipIf import mock +from parameterized import parameterized_class from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import WorkflowFailedError import logging +from aws_lambda_builders.workflows.python_pip.utils import EXPERIMENTAL_FLAG_BUILD_PERFORMANCE + logger = logging.getLogger("aws_lambda_builders.workflows.python_pip.workflow") IS_WINDOWS = platform.system().lower() == "windows" NOT_ARM = platform.processor() != "aarch64" ARM_RUNTIMES = {"python3.8", "python3.9"} +@parameterized_class(("experimental_flags",), [([]), ([EXPERIMENTAL_FLAG_BUILD_PERFORMANCE])]) class TestPythonPipWorkflow(TestCase): """ Verifies that `python_pip` workflow works by building a Lambda that requires Numpy """ TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata") + experimental_flags = [] def setUp(self): self.source_dir = self.TEST_DATA_FOLDER @@ -75,7 +80,12 @@ def check_architecture_in(self, library, architectures): def test_must_build_python_project(self): self.builder.build( - self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime=self.runtime + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_valid, + runtime=self.runtime, + experimental_flags=self.experimental_flags, ) if self.runtime == "python3.6": @@ -100,6 +110,7 @@ def test_must_build_python_project_from_sdist_with_arm(self): self.manifest_path_sdist, runtime=self.runtime, architecture="arm64", + experimental_flags=self.experimental_flags, ) expected_files = self.test_data_files.union({"wrapt", "wrapt-1.13.3.dist-info"}) output_files = set(os.listdir(self.artifacts_dir)) @@ -118,6 +129,7 @@ def test_must_build_python_project_with_arm_architecture(self): self.manifest_path_valid, runtime=self.runtime, architecture="arm64", + experimental_flags=self.experimental_flags, ) expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.20.3.dist-info"}) output_files = set(os.listdir(self.artifacts_dir)) @@ -135,6 +147,7 @@ def test_mismatch_runtime_python_project(self): self.scratch_dir, self.manifest_path_valid, runtime=self.runtime_mismatch[self.runtime], + experimental_flags=self.experimental_flags, ) except WorkflowFailedError as ex: # handle both e.g. missing /usr/bin/python2.7 and situation where @@ -162,7 +175,14 @@ def test_must_resolve_local_dependency(self): # need to make sure the correct path is used in the requirements file locally and in CI with open(manifest, "w") as f: f.write(str(path_to_package)) - self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest, runtime=self.runtime) + self.builder.build( + source_dir, + self.artifacts_dir, + self.scratch_dir, + manifest, + runtime=self.runtime, + experimental_flags=self.experimental_flags, + ) expected_files = { "local_package", "local_package-0.0.0.dist-info", @@ -179,7 +199,12 @@ def test_must_fail_to_resolve_dependencies(self): with self.assertRaises(WorkflowFailedError) as ctx: self.builder.build( - self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_invalid, runtime=self.runtime + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_invalid, + runtime=self.runtime, + experimental_flags=self.experimental_flags, ) message_in_exception = "Invalid requirement: 'boto3=1.19.99'" in str(ctx.exception) @@ -193,6 +218,7 @@ def test_must_log_warning_if_requirements_not_found(self): self.scratch_dir, os.path.join("non", "existent", "manifest"), runtime=self.runtime, + experimental_flags=self.experimental_flags, ) expected_files = self.test_data_files output_files = set(os.listdir(self.artifacts_dir)) @@ -218,6 +244,7 @@ def test_without_download_dependencies_with_dependencies_dir(self): runtime=self.runtime, download_dependencies=False, dependencies_dir=self.dependencies_dir, + experimental_flags=self.experimental_flags, ) # if download_dependencies is False and dependencies is empty, the artifacts_dir should just copy files from @@ -243,6 +270,7 @@ def test_with_download_dependencies_and_dependencies_dir(self): runtime=self.runtime, download_dependencies=True, dependencies_dir=self.dependencies_dir, + experimental_flags=self.experimental_flags, ) # build artifact should be same as usual @@ -287,6 +315,7 @@ def test_without_download_dependencies_without_dependencies_dir(self): runtime=self.runtime, download_dependencies=False, dependencies_dir=None, + experimental_flags=self.experimental_flags, ) # if download_dependencies is False and dependencies is None, the artifacts_dir should just copy files from @@ -318,6 +347,7 @@ def test_without_combine_dependencies(self): download_dependencies=True, dependencies_dir=self.dependencies_dir, combine_dependencies=False, + experimental_flags=self.experimental_flags, ) expected_files = os.listdir(source_dir) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 000000000..97d2ec16f --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,32 @@ +from unittest import TestCase +from unittest.mock import patch + +from aws_lambda_builders import utils + + +class Test_create_symlink_or_copy(TestCase): + @patch("aws_lambda_builders.utils.Path") + @patch("aws_lambda_builders.utils.os") + @patch("aws_lambda_builders.utils.copytree") + def test_must_create_symlink_with_absolute_path(self, patched_copy_tree, pathced_os, patched_path): + source_path = "source/path" + destination_path = "destination/path" + utils.create_symlink_or_copy(source_path, destination_path) + + pathced_os.symlink.assert_called_with( + patched_path(source_path).absolute(), patched_path(destination_path).absolute() + ) + patched_copy_tree.assert_not_called() + + @patch("aws_lambda_builders.utils.Path") + @patch("aws_lambda_builders.utils.os") + @patch("aws_lambda_builders.utils.copytree") + def test_must_copy_if_symlink_fails(self, patched_copy_tree, pathced_os, patched_path): + pathced_os.symlink.side_effect = OSError("Unable to create symlink") + + source_path = "source/path" + destination_path = "destination/path" + utils.create_symlink_or_copy(source_path, destination_path) + + pathced_os.symlink.assert_called_once() + patched_copy_tree.assert_called_with(source_path, destination_path) diff --git a/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py b/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py index fe37bc48a..a00dd468a 100644 --- a/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py +++ b/tests/unit/workflows/nodejs_npm_esbuild/test_workflow.py @@ -1,7 +1,13 @@ from unittest import TestCase from mock import patch, call -from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, CopyDependenciesAction, MoveDependenciesAction +from aws_lambda_builders.actions import ( + CopySourceAction, + CleanUpAction, + CopyDependenciesAction, + MoveDependenciesAction, + LinkSourceAction, +) from aws_lambda_builders.architecture import ARM64 from aws_lambda_builders.workflows.nodejs_npm.actions import NodejsNpmInstallAction, NodejsNpmCIAction from aws_lambda_builders.workflows.nodejs_npm_esbuild import NodejsNpmEsbuildWorkflow @@ -219,7 +225,7 @@ def test_workflow_sets_up_esbuild_actions_without_download_dependencies_with_dep self.assertEqual(len(workflow.actions), 3) self.assertIsInstance(workflow.actions[0], CopySourceAction) - self.assertIsInstance(workflow.actions[1], CopySourceAction) + self.assertIsInstance(workflow.actions[1], LinkSourceAction) self.assertIsInstance(workflow.actions[2], EsbuildBundleAction) def test_workflow_sets_up_esbuild_actions_without_download_dependencies_with_dependencies_dir_no_combine_deps(self): @@ -239,7 +245,7 @@ def test_workflow_sets_up_esbuild_actions_without_download_dependencies_with_dep self.assertEqual(len(workflow.actions), 4) self.assertIsInstance(workflow.actions[0], CopySourceAction) - self.assertIsInstance(workflow.actions[1], CopySourceAction) + self.assertIsInstance(workflow.actions[1], LinkSourceAction) self.assertIsInstance(workflow.actions[2], EsbuildCheckVersionAction) self.assertIsInstance(workflow.actions[3], EsbuildBundleAction) @@ -258,13 +264,14 @@ def test_workflow_sets_up_esbuild_actions_with_download_dependencies_and_depende experimental_flags=[EXPERIMENTAL_FLAG_ESBUILD], ) - self.assertEqual(len(workflow.actions), 5) + self.assertEqual(len(workflow.actions), 6) self.assertIsInstance(workflow.actions[0], CopySourceAction) self.assertIsInstance(workflow.actions[1], NodejsNpmInstallAction) self.assertIsInstance(workflow.actions[2], CleanUpAction) self.assertIsInstance(workflow.actions[3], EsbuildBundleAction) - self.assertIsInstance(workflow.actions[4], CopyDependenciesAction) + self.assertIsInstance(workflow.actions[4], MoveDependenciesAction) + self.assertIsInstance(workflow.actions[5], LinkSourceAction) def test_workflow_sets_up_esbuild_actions_with_download_dependencies_and_dependencies_dir_no_combine_deps(self): workflow = NodejsNpmEsbuildWorkflow( diff --git a/tests/unit/workflows/python_pip/test_workflow.py b/tests/unit/workflows/python_pip/test_workflow.py index b57200efd..e7c66b06f 100644 --- a/tests/unit/workflows/python_pip/test_workflow.py +++ b/tests/unit/workflows/python_pip/test_workflow.py @@ -2,19 +2,36 @@ from mock import patch, ANY, Mock from unittest import TestCase -from aws_lambda_builders.actions import CopySourceAction, CleanUpAction -from aws_lambda_builders.workflows.python_pip.utils import OSUtils +from parameterized import parameterized_class + +from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, LinkSourceAction +from aws_lambda_builders.workflows.python_pip.utils import OSUtils, EXPERIMENTAL_FLAG_BUILD_PERFORMANCE from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator from aws_lambda_builders.workflows.python_pip.workflow import PythonPipBuildAction, PythonPipWorkflow +@parameterized_class( + ("experimental_flags",), + [ + ([]), + ([EXPERIMENTAL_FLAG_BUILD_PERFORMANCE]), + ], +) class TestPythonPipWorkflow(TestCase): + experimental_flags = [] + def setUp(self): self.osutils = OSUtils() self.osutils_mock = Mock(spec=self.osutils) self.osutils_mock.file_exists.return_value = True self.workflow = PythonPipWorkflow( - "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=self.osutils_mock + "source", + "artifacts", + "scratch_dir", + "manifest", + runtime="python3.7", + osutils=self.osutils_mock, + experimental_flags=self.experimental_flags, ) def test_workflow_sets_up_actions(self): @@ -25,7 +42,13 @@ def test_workflow_sets_up_actions(self): def test_workflow_sets_up_actions_without_requirements(self): self.osutils_mock.file_exists.return_value = False self.workflow = PythonPipWorkflow( - "source", "artifacts", "scratch_dir", "manifest", runtime="python3.7", osutils=self.osutils_mock + "source", + "artifacts", + "scratch_dir", + "manifest", + runtime="python3.7", + osutils=self.osutils_mock, + experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 1) self.assertIsInstance(self.workflow.actions[0], CopySourceAction) @@ -46,9 +69,13 @@ def test_workflow_sets_up_actions_without_download_dependencies_with_dependencie osutils=osutils_mock, dependencies_dir="dep", download_dependencies=False, + experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 2) - self.assertIsInstance(self.workflow.actions[0], CopySourceAction) + if self.experimental_flags: + self.assertIsInstance(self.workflow.actions[0], LinkSourceAction) + else: + self.assertIsInstance(self.workflow.actions[0], CopySourceAction) self.assertIsInstance(self.workflow.actions[1], CopySourceAction) def test_workflow_sets_up_actions_with_download_dependencies_and_dependencies_dir(self): @@ -63,14 +90,18 @@ def test_workflow_sets_up_actions_with_download_dependencies_and_dependencies_di osutils=osutils_mock, dependencies_dir="dep", download_dependencies=True, + experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 4) self.assertIsInstance(self.workflow.actions[0], CleanUpAction) self.assertIsInstance(self.workflow.actions[1], PythonPipBuildAction) - self.assertIsInstance(self.workflow.actions[2], CopySourceAction) + if self.experimental_flags: + self.assertIsInstance(self.workflow.actions[2], LinkSourceAction) + else: + self.assertIsInstance(self.workflow.actions[2], CopySourceAction) + # check copying dependencies does not have any exclude + self.assertEqual(self.workflow.actions[2].excludes, []) self.assertIsInstance(self.workflow.actions[3], CopySourceAction) - # check copying dependencies does not have any exclude - self.assertEqual(self.workflow.actions[2].excludes, []) def test_workflow_sets_up_actions_without_download_dependencies_without_dependencies_dir(self): osutils_mock = Mock(spec=self.osutils) @@ -84,6 +115,7 @@ def test_workflow_sets_up_actions_without_download_dependencies_without_dependen osutils=osutils_mock, dependencies_dir=None, download_dependencies=False, + experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 1) self.assertIsInstance(self.workflow.actions[0], CopySourceAction) @@ -101,6 +133,7 @@ def test_workflow_sets_up_actions_without_combine_dependencies(self): dependencies_dir="dep", download_dependencies=True, combine_dependencies=False, + experimental_flags=self.experimental_flags, ) self.assertEqual(len(self.workflow.actions), 3) self.assertIsInstance(self.workflow.actions[0], CleanUpAction)