Skip to content

Build identical artifacts only once #1180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Dunedan wants to merge 1 commit into from
Closed

Build identical artifacts only once #1180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 23 additions & 10 deletions samcli/lib/build/app_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@
Builds the application
"""

import base64
import operator
import os
import io
import json
import logging
from itertools import groupby

try:
import pathlib
Expand Down Expand Up @@ -100,12 +103,23 @@ def build(self):

result = {}

for lambda_function in self._functions_to_build:
grouped_functions = groupby(sorted(self._functions_to_build,
key=operator.attrgetter("runtime", "codeuri", "name")),
operator.attrgetter("runtime", "codeuri"))

LOG.info("Building resource '%s'", lambda_function.name)
result[lambda_function.name] = self._build_function(lambda_function.name,
lambda_function.codeuri,
lambda_function.runtime)
groups = dict()
for group_key, iterator in grouped_functions:
groups[group_key] = list(iterator)

for group_key in groups:
runtime = group_key[0]
codeuri = group_key[1]
LOG.info("Building artifact for runtime %s and CodeUri %s used by the following " +
"resources: %s",
runtime, codeuri, ", ".join([x.name for x in groups[group_key]]))
artifact_path = self._build_function(codeuri, runtime)
for func in groups[group_key]:
result[func.name] = artifact_path

return result

Expand Down Expand Up @@ -152,16 +166,13 @@ def update_template(self, template_dict, original_template_path, built_artifacts

return template_dict

def _build_function(self, function_name, codeuri, runtime):
def _build_function(self, codeuri, runtime):
"""
Given the function information, this method will build the Lambda function. Depending on the configuration
it will either build the function in process or by spinning up a Docker container.

Parameters
----------
function_name : str
Name or LogicalId of the function

codeuri : str
Path to where the code lives

Expand All @@ -181,7 +192,9 @@ def _build_function(self, function_name, codeuri, runtime):
config = get_workflow_config(runtime, code_dir, self._base_dir)

# artifacts directory will be created by the builder
artifacts_dir = str(pathlib.Path(self._build_dir, function_name))
artifacts_dir = str(pathlib.Path(
self._build_dir,
"{}_{}".format(runtime, base64.b64encode(codeuri.encode('utf-8')).decode('utf-8'))))

with osutils.mkdir_temp() as scratch_dir:
manifest_path = self._manifest_path_override or os.path.join(code_dir, config.manifest_name)
Expand Down
103 changes: 82 additions & 21 deletions tests/unit/lib/build_module/test_app_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,41 +1,100 @@

import base64
import os
import docker
import json

from unittest import TestCase
from mock import Mock, call, patch

from samcli.commands.local.lib.provider import Function
from samcli.lib.build.app_builder import ApplicationBuilder,\
UnsupportedBuilderLibraryVersionError, BuildError, \
LambdaBuilderError, ContainerBuildNotSupported


class TestApplicationBuilder_build(TestCase):

def setUp(self):
self.func1 = Mock()
self.func2 = Mock()
self.builder = ApplicationBuilder([self.func1, self.func2],
"builddir",
"basedir")

def test_must_iterate_on_functions(self):
def test_artifacts_for_identical_functions(self):
func1 = Function(name="function_name1",
runtime="runtime",
memory=1234,
timeout=12,
handler="handler",
codeuri="codeuri",
environment=None,
rolearn=None,
layers=[])
func2 = Function(name="function_name2",
runtime="runtime",
memory=1234,
timeout=12,
handler="handler",
codeuri="codeuri",
environment=None,
rolearn=None,
layers=[])

builder = ApplicationBuilder([func1, func2], "builddir", "basedir")
build_function_mock = Mock()
builder._build_function = build_function_mock

self.builder._build_function = build_function_mock
result = builder.build()

result = self.builder.build()
self.assertEquals(result, {
func1.name: build_function_mock.return_value,
func2.name: build_function_mock.return_value,
})

build_function_mock.assert_has_calls([
call(func1.codeuri, func1.runtime)], any_order=False)

def test_artifacts_for_differing_functions(self):
func1 = Function(name="function_name1",
runtime="runtime",
memory=1234,
timeout=12,
handler="handler",
codeuri="codeuri",
environment=None,
rolearn=None,
layers=[])
func2 = Function(name="function_name2",
runtime="runtime2",
memory=1234,
timeout=12,
handler="handler",
codeuri="codeuri",
environment=None,
rolearn=None,
layers=[])
func3 = Function(name="function_name3",
runtime="runtime3",
memory=1234,
timeout=12,
handler="handler",
codeuri="codeuri3",
environment=None,
rolearn=None,
layers=[])

builder = ApplicationBuilder([func1, func2, func3], "builddir", "basedir")
build_function_mock = Mock()
builder._build_function = build_function_mock

result = builder.build()

self.assertEquals(result, {
self.func1.name: build_function_mock.return_value,
self.func2.name: build_function_mock.return_value,
func1.name: build_function_mock.return_value,
func2.name: build_function_mock.return_value,
func3.name: build_function_mock.return_value,
})

build_function_mock.assert_has_calls([
call(self.func1.name, self.func1.codeuri, self.func1.runtime),
call(self.func2.name, self.func2.codeuri, self.func2.runtime),
], any_order=False)
call(func1.codeuri, func1.runtime),
call(func2.codeuri, func2.runtime),
call(func3.codeuri, func3.runtime),
], any_order=True)


class TestApplicationBuilder_update_template(TestCase):
Expand Down Expand Up @@ -116,10 +175,13 @@ def setUp(self):
"/build/dir",
"/base/dir")

@staticmethod
def _build_artifacts_dir(codeuri, runtime):
return "{}_{}".format(runtime, base64.b64encode(codeuri.encode('utf-8')).decode('utf-8'))

@patch("samcli.lib.build.app_builder.get_workflow_config")
@patch("samcli.lib.build.app_builder.osutils")
def test_must_build_in_process(self, osutils_mock, get_workflow_config_mock):
function_name = "function_name"
codeuri = "path/to/source"
runtime = "runtime"
scratch_dir = "scratch"
Expand All @@ -132,10 +194,10 @@ def test_must_build_in_process(self, osutils_mock, get_workflow_config_mock):
self.builder._build_function_in_process = Mock()

code_dir = "/base/dir/path/to/source"
artifacts_dir = "/build/dir/function_name"
artifacts_dir = os.path.join("/build/dir/", self._build_artifacts_dir(codeuri, runtime))
manifest_path = os.path.join(code_dir, config_mock.manifest_name)

self.builder._build_function(function_name, codeuri, runtime)
self.builder._build_function(codeuri, runtime)

self.builder._build_function_in_process.assert_called_with(config_mock,
code_dir,
Expand All @@ -147,7 +209,6 @@ def test_must_build_in_process(self, osutils_mock, get_workflow_config_mock):
@patch("samcli.lib.build.app_builder.get_workflow_config")
@patch("samcli.lib.build.app_builder.osutils")
def test_must_build_in_container(self, osutils_mock, get_workflow_config_mock):
function_name = "function_name"
codeuri = "path/to/source"
runtime = "runtime"
scratch_dir = "scratch"
Expand All @@ -160,12 +221,12 @@ def test_must_build_in_container(self, osutils_mock, get_workflow_config_mock):
self.builder._build_function_on_container = Mock()

code_dir = "/base/dir/path/to/source"
artifacts_dir = "/build/dir/function_name"
artifacts_dir = os.path.join("/build/dir/", self._build_artifacts_dir(codeuri, runtime))
manifest_path = os.path.join(code_dir, config_mock.manifest_name)

# Settting the container manager will make us use the container
self.builder._container_manager = Mock()
self.builder._build_function(function_name, codeuri, runtime)
self.builder._build_function(codeuri, runtime)

self.builder._build_function_on_container.assert_called_with(config_mock,
code_dir,
Expand Down