feat: use symlink instead of copy for some build operations

aws_lambda_builders/actions.py

@@ -5,8 +5,10 @@
 import logging
 import os
 import shutil
+from pathlib import Path
 from typing import Set, Iterator, Tuple
 
+from aws_lambda_builders import utils
 from aws_lambda_builders.utils import copytree
 
 LOG = logging.getLogger(__name__)
@@ -31,6 +33,9 @@ class Purpose(object):
     # Action is copying source code
     COPY_SOURCE = "COPY_SOURCE"
 
+    # Action is linking source code
+    LINK_SOURCE = "LINK_SOURCE"
+
     # Action is copying dependencies
     COPY_DEPENDENCIES = "COPY_DEPENDENCIES"
 
@@ -111,6 +116,31 @@ def execute(self):
         copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes))
 
 
+class LinkSourceAction(BaseAction):
+
+    NAME = "LinkSource"
+
+    DESCRIPTION = "Linking source code to the target folder"
+
+    PURPOSE = Purpose.LINK_SOURCE
+
+    def __init__(self, source_dir, dest_dir):
+        self._source_dir = source_dir
+        self._dest_dir = dest_dir
+
+    def execute(self):
+        source_files = set(os.listdir(self._source_dir))
+
+        for source_file in source_files:
+            source_path = Path(self._source_dir, source_file)
+            destination_path = Path(self._dest_dir, source_file)
+            if destination_path.exists():
+                os.remove(destination_path)
+            else:
+                os.makedirs(destination_path.parent, exist_ok=True)
+            utils.create_symlink_or_copy(str(source_path), str(destination_path))
+
+
 class CopyDependenciesAction(BaseAction):
 
     NAME = "CopyDependencies"
@@ -175,10 +205,10 @@ def __init__(self, target_dir):
 
     def execute(self):
         if not os.path.isdir(self.target_dir):
-            LOG.info("Clean up action: %s does not exist and will be skipped.", str(self.target_dir))
+            LOG.debug("Clean up action: %s does not exist and will be skipped.", str(self.target_dir))
             return
         targets = os.listdir(self.target_dir)
-        LOG.info("Clean up action: folder %s will be cleaned", str(self.target_dir))
+        LOG.debug("Clean up action: folder %s will be cleaned", str(self.target_dir))
 
         for name in targets:
             target_path = os.path.join(self.target_dir, name)

aws_lambda_builders/utils.py

@@ -6,6 +6,7 @@
 import sys
 import os
 import logging
+from pathlib import Path
 
 from aws_lambda_builders.architecture import X86_64, ARM64
 
@@ -182,3 +183,16 @@ def get_goarch(architecture):
         returns a valid GO Architecture value
     """
     return "arm64" if architecture == ARM64 else "amd64"
+
+
+def create_symlink_or_copy(source: str, destination: str) -> None:
+    """Tries to create symlink, if it fails it will copy source into destination"""
+    LOG.debug("Creating symlink; source: %s, destination: %s", source, destination)
+    try:
+        os.symlink(Path(source).absolute(), Path(destination).absolute())
+    except OSError as ex:
+        LOG.warning(
+            "Symlink operation is failed, falling back to copying files",
+            exc_info=ex if LOG.isEnabledFor(logging.DEBUG) else None,
+        )
+        copytree(source, destination)

aws_lambda_builders/workflows/nodejs_npm_esbuild/utils.py

@@ -1,11 +1,12 @@
 """
 esbuild specific utilities and feature flag
 """
+from typing import Optional, List
 
 EXPERIMENTAL_FLAG_ESBUILD = "experimentalEsbuild"
 
 
-def is_experimental_esbuild_scope(experimental_flags):
+def is_experimental_esbuild_scope(experimental_flags: Optional[List[str]]) -> bool:
     """
     A function which will determine if experimental esbuild scope is active
     """

aws_lambda_builders/workflows/nodejs_npm_esbuild/workflow.py

@@ -10,9 +10,9 @@
 from aws_lambda_builders.actions import (
     CopySourceAction,
     CleanUpAction,
-    CopyDependenciesAction,
     MoveDependenciesAction,
     BaseAction,
+    LinkSourceAction,
 )
 from aws_lambda_builders.utils import which
 from .actions import (
@@ -168,13 +168,18 @@ def _accelerate_workflow_actions(
             actions += [install_action, CleanUpAction(self.dependencies_dir)]
             if self.combine_dependencies:
                 # Auto dependency layer disabled, first build
-                actions += [esbuild_with_deps, CopyDependenciesAction(source_dir, scratch_dir, self.dependencies_dir)]
+                actions += [
+                    esbuild_with_deps,
+                    MoveDependenciesAction(source_dir, scratch_dir, self.dependencies_dir),
+                    LinkSourceAction(self.dependencies_dir, scratch_dir),
+                ]
             else:
                 # Auto dependency layer enabled, first build
                 actions += esbuild_no_deps + [MoveDependenciesAction(source_dir, scratch_dir, self.dependencies_dir)]
         else:
             if self.dependencies_dir:
-                actions.append(CopySourceAction(self.dependencies_dir, scratch_dir))
+                actions.append(LinkSourceAction(self.dependencies_dir, scratch_dir))
+
                 if self.combine_dependencies:
                     # Auto dependency layer disabled, subsequent builds
                     actions += [esbuild_with_deps]

aws_lambda_builders/workflows/python_pip/utils.py

@@ -11,6 +11,9 @@
 import tarfile
 import subprocess
 import sys
+from typing import Optional, List
+
+EXPERIMENTAL_FLAG_BUILD_PERFORMANCE = "experimentalBuildPerformance"
 
 
 class OSUtils(object):
@@ -106,3 +109,10 @@ def pipe(self):
     def basename(self, path):
         # type: (str) -> str
         return os.path.basename(path)
+
+
+def is_experimental_build_improvements_enabled(experimental_flags: Optional[List[str]]) -> bool:
+    """
+    A function which will determine if experimental build improvements is active
+    """
+    return bool(experimental_flags) and EXPERIMENTAL_FLAG_BUILD_PERFORMANCE in experimental_flags

aws_lambda_builders/workflows/python_pip/workflow.py

@@ -4,11 +4,11 @@
 import logging
 
 from aws_lambda_builders.workflow import BaseWorkflow, Capability
-from aws_lambda_builders.actions import CopySourceAction, CleanUpAction
+from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, LinkSourceAction
 from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator
 
 from .actions import PythonPipBuildAction
-from .utils import OSUtils
+from .utils import OSUtils, is_experimental_build_improvements_enabled
 
 LOG = logging.getLogger(__name__)
 
@@ -106,7 +106,10 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
         # folder
         if self.dependencies_dir and self.combine_dependencies:
             # when copying downloaded dependencies back to artifacts folder, don't exclude anything
-            self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir))
+            if is_experimental_build_improvements_enabled(self.experimental_flags):
+                self.actions.append(LinkSourceAction(self.dependencies_dir, artifacts_dir))
+            else:
+                self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir))
 
         self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES))
 

tests/integration/workflows/python_pip/test_python_pip.py

@@ -5,23 +5,28 @@
 import tempfile
 from unittest import TestCase, skipIf
 import mock
+from parameterized import parameterized_class
 
 from aws_lambda_builders.builder import LambdaBuilder
 from aws_lambda_builders.exceptions import WorkflowFailedError
 import logging
 
+from aws_lambda_builders.workflows.python_pip.utils import EXPERIMENTAL_FLAG_BUILD_PERFORMANCE
+
 logger = logging.getLogger("aws_lambda_builders.workflows.python_pip.workflow")
 IS_WINDOWS = platform.system().lower() == "windows"
 NOT_ARM = platform.processor() != "aarch64"
 ARM_RUNTIMES = {"python3.8", "python3.9"}
 
 
+@parameterized_class(("experimental_flags",), [([]), ([EXPERIMENTAL_FLAG_BUILD_PERFORMANCE])])
 class TestPythonPipWorkflow(TestCase):
     """
     Verifies that `python_pip` workflow works by building a Lambda that requires Numpy
     """
 
     TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
+    experimental_flags = []
 
     def setUp(self):
         self.source_dir = self.TEST_DATA_FOLDER
@@ -75,7 +80,12 @@ def check_architecture_in(self, library, architectures):
 
     def test_must_build_python_project(self):
         self.builder.build(
-            self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime=self.runtime
+            self.source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            self.manifest_path_valid,
+            runtime=self.runtime,
+            experimental_flags=self.experimental_flags,
         )
 
         if self.runtime == "python3.6":
@@ -100,6 +110,7 @@ def test_must_build_python_project_from_sdist_with_arm(self):
             self.manifest_path_sdist,
             runtime=self.runtime,
             architecture="arm64",
+            experimental_flags=self.experimental_flags,
         )
         expected_files = self.test_data_files.union({"wrapt", "wrapt-1.13.3.dist-info"})
         output_files = set(os.listdir(self.artifacts_dir))
@@ -118,6 +129,7 @@ def test_must_build_python_project_with_arm_architecture(self):
             self.manifest_path_valid,
             runtime=self.runtime,
             architecture="arm64",
+            experimental_flags=self.experimental_flags,
         )
         expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.20.3.dist-info"})
         output_files = set(os.listdir(self.artifacts_dir))
@@ -135,6 +147,7 @@ def test_mismatch_runtime_python_project(self):
                 self.scratch_dir,
                 self.manifest_path_valid,
                 runtime=self.runtime_mismatch[self.runtime],
+                experimental_flags=self.experimental_flags,
             )
         except WorkflowFailedError as ex:
             # handle both e.g. missing /usr/bin/python2.7 and situation where
@@ -162,7 +175,14 @@ def test_must_resolve_local_dependency(self):
         # need to make sure the correct path is used in the requirements file locally and in CI
         with open(manifest, "w") as f:
             f.write(str(path_to_package))
-        self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest, runtime=self.runtime)
+        self.builder.build(
+            source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            manifest,
+            runtime=self.runtime,
+            experimental_flags=self.experimental_flags,
+        )
         expected_files = {
             "local_package",
             "local_package-0.0.0.dist-info",
@@ -179,7 +199,12 @@ def test_must_fail_to_resolve_dependencies(self):
 
         with self.assertRaises(WorkflowFailedError) as ctx:
             self.builder.build(
-                self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_invalid, runtime=self.runtime
+                self.source_dir,
+                self.artifacts_dir,
+                self.scratch_dir,
+                self.manifest_path_invalid,
+                runtime=self.runtime,
+                experimental_flags=self.experimental_flags,
             )
 
         message_in_exception = "Invalid requirement: 'boto3=1.19.99'" in str(ctx.exception)
@@ -193,6 +218,7 @@ def test_must_log_warning_if_requirements_not_found(self):
                 self.scratch_dir,
                 os.path.join("non", "existent", "manifest"),
                 runtime=self.runtime,
+                experimental_flags=self.experimental_flags,
             )
         expected_files = self.test_data_files
         output_files = set(os.listdir(self.artifacts_dir))
@@ -218,6 +244,7 @@ def test_without_download_dependencies_with_dependencies_dir(self):
             runtime=self.runtime,
             download_dependencies=False,
             dependencies_dir=self.dependencies_dir,
+            experimental_flags=self.experimental_flags,
         )
 
         # if download_dependencies is False and dependencies is empty, the artifacts_dir should just copy files from
@@ -243,6 +270,7 @@ def test_with_download_dependencies_and_dependencies_dir(self):
             runtime=self.runtime,
             download_dependencies=True,
             dependencies_dir=self.dependencies_dir,
+            experimental_flags=self.experimental_flags,
         )
 
         # build artifact should be same as usual
@@ -287,6 +315,7 @@ def test_without_download_dependencies_without_dependencies_dir(self):
                 runtime=self.runtime,
                 download_dependencies=False,
                 dependencies_dir=None,
+                experimental_flags=self.experimental_flags,
             )
 
         # if download_dependencies is False and dependencies is None, the artifacts_dir should just copy files from
@@ -318,6 +347,7 @@ def test_without_combine_dependencies(self):
             download_dependencies=True,
             dependencies_dir=self.dependencies_dir,
             combine_dependencies=False,
+            experimental_flags=self.experimental_flags,
         )
 
         expected_files = os.listdir(source_dir)

tests/unit/test_utils.py

@@ -0,0 +1,32 @@
+from unittest import TestCase
+from unittest.mock import patch
+
+from aws_lambda_builders import utils
+
+
+class Test_create_symlink_or_copy(TestCase):
+    @patch("aws_lambda_builders.utils.Path")
+    @patch("aws_lambda_builders.utils.os")
+    @patch("aws_lambda_builders.utils.copytree")
+    def test_must_create_symlink_with_absolute_path(self, patched_copy_tree, pathced_os, patched_path):
+        source_path = "source/path"
+        destination_path = "destination/path"
+        utils.create_symlink_or_copy(source_path, destination_path)
+
+        pathced_os.symlink.assert_called_with(
+            patched_path(source_path).absolute(), patched_path(destination_path).absolute()
+        )
+        patched_copy_tree.assert_not_called()
+
+    @patch("aws_lambda_builders.utils.Path")
+    @patch("aws_lambda_builders.utils.os")
+    @patch("aws_lambda_builders.utils.copytree")
+    def test_must_copy_if_symlink_fails(self, patched_copy_tree, pathced_os, patched_path):
+        pathced_os.symlink.side_effect = OSError("Unable to create symlink")
+
+        source_path = "source/path"
+        destination_path = "destination/path"
+        utils.create_symlink_or_copy(source_path, destination_path)
+
+        pathced_os.symlink.assert_called_once()
+        patched_copy_tree.assert_called_with(source_path, destination_path)