Merge develop to main

.github/workflows/pr-labeler.yml

@@ -21,7 +21,7 @@ jobs:
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
-            const maintainers = ['jfuss', 'c2tarun', 'hoffa', 'awood45', 'CoshUS', 'aahung', 'hawflau', 'mndeveci', 'ssenchenko', 'wchengru', 'mingkun2020', 'qingchm', 'moelasmar', 'xazhao', 'mildaniel', 'marekaiv', 'torresxb1']
+            const maintainers = ['jfuss', 'hoffa', 'awood45', 'aahung', 'hawflau', 'mndeveci', 'ssenchenko', 'qingchm', 'moelasmar', 'xazhao', 'mildaniel', 'marekaiv', 'torresxb1', 'lucashuy', 'hnnasit', 'sriram-mv']
             if (maintainers.includes(context.payload.sender.login)) {
               github.rest.issues.addLabels({
                 issue_number: context.issue.number,

aws_lambda_builders/__init__.py

@@ -1,5 +1,8 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "1.19.0"
+
+# Changing version will trigger a new release!
+# Please make the version change as the last step of your development.
+__version__ = "1.20.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/workflows/go_modules/builder.py

@@ -21,7 +21,7 @@ class GoModulesBuilder(object):
 
     LANGUAGE = "go"
 
-    def __init__(self, osutils, binaries, mode=BuildMode.RELEASE, architecture=X86_64):
+    def __init__(self, osutils, binaries, mode=BuildMode.RELEASE, architecture=X86_64, trim_go_path=False):
         """Initialize a GoModulesBuilder.
 
         :type osutils: :class:`lambda_builders.utils.OSUtils`
@@ -33,11 +33,15 @@ def __init__(self, osutils, binaries, mode=BuildMode.RELEASE, architecture=X86_6
 
         :type architecture: str
         :param architecture: name of the type of architecture
+
+        :type trim_go_path: bool
+        :param trim_go_path: should go build use -trimpath flag
         """
         self.osutils = osutils
         self.binaries = binaries
         self.mode = mode
         self.goarch = get_goarch(architecture)
+        self.trim_go_path = trim_go_path
 
     def build(self, source_dir_path, output_path):
         """Builds a go project onto an output path.
@@ -53,6 +57,9 @@ def build(self, source_dir_path, output_path):
         env.update({"GOOS": "linux", "GOARCH": self.goarch})
         runtime_path = self.binaries[self.LANGUAGE].binary_path
         cmd = [runtime_path, "build"]
+        if self.trim_go_path:
+            LOG.debug("Trimpath requested: Setting go build configuration to -trimpath")
+            cmd += ["-trimpath"]
         if self.mode and self.mode.lower() == BuildMode.DEBUG:
             LOG.debug("Debug build requested: Setting configuration to Debug")
             cmd += ["-gcflags", "all=-N -l"]

aws_lambda_builders/workflows/go_modules/workflow.py

@@ -28,10 +28,13 @@ def __init__(
 
         options = kwargs.get("options") or {}
         handler = options.get("artifact_executable_name", None)
+        trim_go_path = options.get("trim_go_path", False)
 
         output_path = osutils.joinpath(artifacts_dir, handler)
 
-        builder = GoModulesBuilder(osutils, binaries=self.binaries, mode=mode, architecture=self.architecture)
+        builder = GoModulesBuilder(
+            osutils, binaries=self.binaries, mode=mode, architecture=self.architecture, trim_go_path=trim_go_path
+        )
         self.actions = [GoModulesBuildAction(source_dir, output_path, builder)]
 
     def get_validators(self):

aws_lambda_builders/workflows/python_pip/workflow.py

@@ -109,7 +109,8 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
         # folder
         if self.dependencies_dir and self.combine_dependencies:
             # when copying downloaded dependencies back to artifacts folder, don't exclude anything
-            if is_experimental_build_improvements_enabled(self.experimental_flags):
+            # symlinking python dependencies is disabled for now since it is breaking sam local commands
+            if False and is_experimental_build_improvements_enabled(self.experimental_flags):
                 self.actions.append(LinkSourceAction(self.dependencies_dir, artifacts_dir))
             else:
                 self.actions.append(CopySourceAction(self.dependencies_dir, artifacts_dir))

tests/integration/workflows/go_modules/test_go.py

@@ -13,6 +13,7 @@
 from aws_lambda_builders.exceptions import WorkflowFailedError
 
 from tests.integration.workflows.go_modules.utils import get_executable_arch
+from tests.integration.workflows.go_modules.utils import get_md5_hexdigest
 
 
 class TestGoWorkflow(TestCase):
@@ -112,6 +113,71 @@ def test_builds_arm64_architecture(self):
             options={"artifact_executable_name": "no-deps-main-arm64"},
             architecture="arm64",
         )
-
         pathname = Path(self.artifacts_dir, "no-deps-main-arm64")
         self.assertEqual(get_executable_arch(pathname), "AArch64")
+
+    def test_builds_with_trimpath(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
+        built_trimpath = self.builder.build(
+            source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            os.path.join(source_dir, "go.mod"),
+            runtime=self.runtime,
+            options={"artifact_executable_name": "no-deps-main-trimpath", "trim_go_path": True},
+            architecture="x86_64",
+        )
+        pathname = Path(self.artifacts_dir, "no-deps-main-trimpath")
+        self.assertEqual(get_executable_arch(pathname), "x64")
+
+    def test_builds_without_trimpath_are_not_equal(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
+        built_no_trimpath = self.builder.build(
+            source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            os.path.join(source_dir, "go.mod"),
+            runtime=self.runtime,
+            options={"artifact_executable_name": "no-deps-main", "trim_go_path": False},
+            architecture="x86_64",
+        )
+
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps-copy")
+        built_no_trimpath_copy = self.builder.build(
+            source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            os.path.join(source_dir, "go.mod"),
+            runtime=self.runtime,
+            options={"artifact_executable_name": "no-deps-main-copy", "trim_go_path": False},
+            architecture="x86_64",
+        )
+        pathname = Path(self.artifacts_dir, "no-deps-main")
+        pathnameOfCopy = Path(self.artifacts_dir, "no-deps-main-copy")
+        self.assertNotEqual(get_md5_hexdigest(pathname), get_md5_hexdigest(pathnameOfCopy))
+
+    def test_builds_with_trimpath_are_equal(self):
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
+        built_with_trimpath = self.builder.build(
+            source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            os.path.join(source_dir, "go.mod"),
+            runtime=self.runtime,
+            options={"artifact_executable_name": "no-deps-main", "trim_go_path": True},
+            architecture="x86_64",
+        )
+
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps-copy")
+        built_with_trimpath_copy = self.builder.build(
+            source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            os.path.join(source_dir, "go.mod"),
+            runtime=self.runtime,
+            options={"artifact_executable_name": "no-deps-main-copy", "trim_go_path": True},
+            architecture="x86_64",
+        )
+        pathname = Path(self.artifacts_dir, "no-deps-main")
+        pathnameOfCopy = Path(self.artifacts_dir, "no-deps-main-copy")
+        self.assertEqual(get_md5_hexdigest(pathname), get_md5_hexdigest(pathnameOfCopy))

tests/integration/workflows/go_modules/testdata/no-deps-copy/go.mod

@@ -0,0 +1 @@
+module github.com/awslabs/aws-lambda-builders

tests/integration/workflows/go_modules/testdata/no-deps-copy/main.go

@@ -0,0 +1,4 @@
+package main
+
+func main() {
+}

tests/integration/workflows/go_modules/utils.py

@@ -1,4 +1,5 @@
 from elftools.elf.elffile import ELFFile
+import hashlib
 
 
 def get_executable_arch(path):
@@ -18,3 +19,22 @@ def get_executable_arch(path):
     with open(str(path), "rb") as f:
         e = ELFFile(f)
         return e.get_machine_arch()
+
+
+def get_md5_hexdigest(path):
+    """
+    Returns the hexdigest of a binary
+
+    Parameters
+    ----------
+    path : str
+        path to the Go binaries generated
+
+    Returns
+    -------
+    str
+        Hex digest of the binaries
+    """
+    with open(str(path), "rb") as f:
+        hashed = hashlib.md5(f.read())
+        return hashed.hexdigest()

tests/unit/workflows/go_modules/test_builder.py

@@ -61,6 +61,17 @@ def test_debug_configuration_set(self):
             stdout="PIPE",
         )
 
+    def test_trimpath_configuration_set(self):
+        self.under_test = GoModulesBuilder(self.osutils, self.binaries, "release", "x86_64", True)
+        self.under_test.build("source_dir", "output_path")
+        self.osutils.popen.assert_called_with(
+            ["/path/to/go", "build", "-trimpath", "-o", "output_path", "source_dir"],
+            cwd="source_dir",
+            env={"GOOS": "linux", "GOARCH": "amd64"},
+            stderr="PIPE",
+            stdout="PIPE",
+        )
+
     def test_debug_configuration_set_with_arm_architecture(self):
         self.under_test = GoModulesBuilder(self.osutils, self.binaries, "Debug", "arm64")
         self.under_test.build("source_dir", "output_path")

tests/unit/workflows/python_pip/test_workflow.py

@@ -88,7 +88,8 @@ def test_workflow_sets_up_actions_without_download_dependencies_with_dependencie
             experimental_flags=self.experimental_flags,
         )
         self.assertEqual(len(self.workflow.actions), 2)
-        if self.experimental_flags:
+        # symlinking python dependencies is disabled for now since it is breaking sam local commands
+        if False and self.experimental_flags:
             self.assertIsInstance(self.workflow.actions[0], LinkSourceAction)
         else:
             self.assertIsInstance(self.workflow.actions[0], CopySourceAction)
@@ -111,7 +112,8 @@ def test_workflow_sets_up_actions_with_download_dependencies_and_dependencies_di
         self.assertEqual(len(self.workflow.actions), 4)
         self.assertIsInstance(self.workflow.actions[0], CleanUpAction)
         self.assertIsInstance(self.workflow.actions[1], PythonPipBuildAction)
-        if self.experimental_flags:
+        # symlinking python dependencies is disabled for now since it is breaking sam local commands
+        if False and self.experimental_flags:
             self.assertIsInstance(self.workflow.actions[2], LinkSourceAction)
         else:
             self.assertIsInstance(self.workflow.actions[2], CopySourceAction)