From 8ed45fdc7634e06f729d12ac56a2460982b50963 Mon Sep 17 00:00:00 2001 From: Setu Shah Date: Thu, 24 Jun 2021 21:42:03 -0700 Subject: [PATCH 1/8] feat(lambda-python): add support for `buildArgs` --- .../aws-lambda-python/lib/bundling.ts | 10 +++++++++- .../aws-lambda-python/lib/function.ts | 8 ++++++++ .../@aws-cdk/aws-lambda-python/lib/layer.ts | 8 ++++++++ .../aws-lambda-python/test/bundling.test.ts | 20 +++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts index 5a3228a78fef7..277fd6c0d5cd0 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts @@ -71,13 +71,20 @@ export interface BundlingOptions { * @default - based on `assetHashType` */ readonly assetHash?: string; + + /** + * Build arguments to pass when building the bundling image. + * + * @default - no build arguments are passed + */ + readonly buildArgs?: { [key: string]: string }; } /** * Produce bundled Lambda asset code */ export function bundle(options: BundlingOptions): lambda.Code { - const { entry, runtime, outputPathSuffix } = options; + const { entry, runtime, outputPathSuffix, buildArgs } = options; const stagedir = cdk.FileSystem.mkdtemp('python-bundling-'); const hasDeps = stageDependencies(entry, stagedir); @@ -100,6 +107,7 @@ export function bundle(options: BundlingOptions): lambda.Code { const image = cdk.DockerImage.fromBuild(stagedir, { buildArgs: { + ...buildArgs, IMAGE: runtime.bundlingDockerImage.image, }, file: dockerfile, diff --git a/packages/@aws-cdk/aws-lambda-python/lib/function.ts b/packages/@aws-cdk/aws-lambda-python/lib/function.ts index 733c115c0383d..7e14484c0bdd3 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/function.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/function.ts @@ -77,6 +77,13 @@ export interface PythonFunctionProps extends lambda.FunctionOptions { * @default - based on `assetHashType` */ readonly assetHash?: string; + + /** + * Build arguments to pass when building the bundling image. + * + * @default - no build arguments are passed + */ + readonly buildArgs?: { [key: string]: string }; } /** @@ -112,6 +119,7 @@ export class PythonFunction extends lambda.Function { outputPathSuffix: '.', assetHashType: props.assetHashType, assetHash: props.assetHash, + buildArgs: props.buildArgs, }), handler: `${index.slice(0, -3)}.${handler}`, }); diff --git a/packages/@aws-cdk/aws-lambda-python/lib/layer.ts b/packages/@aws-cdk/aws-lambda-python/lib/layer.ts index 1a9684e224580..d34ac13b723af 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/layer.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/layer.ts @@ -21,6 +21,13 @@ export interface PythonLayerVersionProps extends lambda.LayerVersionOptions { * @default - All runtimes are supported. */ readonly compatibleRuntimes?: lambda.Runtime[]; + + /** + * Build arguments to pass when building the bundling image. + * + * @default - no build arguments are passed + */ + readonly buildArgs?: { [key: string]: string }; } /** @@ -50,6 +57,7 @@ export class PythonLayerVersion extends lambda.LayerVersion { entry, runtime, outputPathSuffix: 'python', + buildArgs: props.buildArgs, }), }); } diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index d6de4ee23aedd..cb03051eeb195 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -42,6 +42,26 @@ test('Bundling a function without dependencies', () => { })); }); +test('Bundling Docker with build args', () => { + const entry = path.join(__dirname, 'lambda-handler-nodeps'); + bundle({ + entry, + runtime: Runtime.PYTHON_3_7, + outputPathSuffix: '.', + buildArgs: { + HELLO: 'WORLD', + }, + }); + + expect(Code.fromAsset).toHaveBeenCalledWith(entry, + expect.objectContaining({ + buildArgs: expect.objectContaining({ + HELLO: 'WORLD', + }), + })); +}); + + test('Bundling a function with requirements.txt installed', () => { const entry = path.join(__dirname, 'lambda-handler'); bundle({ From 0ca056a2ea727d44dc16624676108a4e5a5b69dc Mon Sep 17 00:00:00 2001 From: Setu Shah Date: Thu, 24 Jun 2021 22:01:03 -0700 Subject: [PATCH 2/8] Add decsription about `buildArgs` to readme --- packages/@aws-cdk/aws-lambda-python/README.md | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index ffd19568aa5dc..f318655ca4b8b 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -28,14 +28,24 @@ Define a `PythonFunction`: import * as lambda from "@aws-cdk/aws-lambda"; import { PythonFunction } from "@aws-cdk/aws-lambda-python"; -new PythonFunction(this, 'MyFunction', { - entry: '/path/to/my/function', // required - index: 'my_index.py', // optional, defaults to 'index.py' - handler: 'my_exported_func', // optional, defaults to 'handler' +new PythonFunction(this, "MyFunction", { + entry: "/path/to/my/function", // required + index: "my_index.py", // optional, defaults to 'index.py' + handler: "my_exported_func", // optional, defaults to 'handler' runtime: lambda.Runtime.PYTHON_3_6, // optional, defaults to lambda.Runtime.PYTHON_3_7 }); ``` +Additional `buildArgs` to use when bundling the Docker building image can be provided as: + +```ts +new PythonFunction(this, "my-handler", { + buildArgs: { + HTTPS_PROXY: "https://127.0.0.1:3001", + }, +}); +``` + All other properties of `lambda.Function` are supported, see also the [AWS Lambda construct library](https://github.com/aws/aws-cdk/tree/master/packages/%40aws-cdk/aws-lambda). ## Module Dependencies @@ -44,9 +54,9 @@ If `requirements.txt` or `Pipfile` exists at the entry path, the construct will all required modules in a [Lambda compatible Docker container](https://gallery.ecr.aws/sam/build-python3.7) according to the `runtime`. -Python bundles are only recreated and published when a file in a source directory has changed. +Python bundles are only recreated and published when a file in a source directory has changed. Therefore (and as a general best-practice), it is highly recommended to commit a lockfile with a -list of all transitive dependencies and their exact versions. +list of all transitive dependencies and their exact versions. This will ensure that when any dependency version is updated, the bundle asset is recreated and uploaded. To that end, we recommend using [`pipenv`] or [`poetry`] which has lockfile support. @@ -87,11 +97,11 @@ or `Pipfile` or `poetry.lock` with the associated `pyproject.toml` at the entry layer. ```ts -new lambda.PythonFunction(this, 'MyFunction', { - entry: '/path/to/my/function', +new lambda.PythonFunction(this, "MyFunction", { + entry: "/path/to/my/function", layers: [ - new lambda.PythonLayerVersion(this, 'MyLayer', { - entry: '/path/to/my/layer', // point this to your library's directory + new lambda.PythonLayerVersion(this, "MyLayer", { + entry: "/path/to/my/layer", // point this to your library's directory }), ], }); From b4bb4543a8e674ceb25fbe249ce477e935fab656 Mon Sep 17 00:00:00 2001 From: Setu Shah Date: Thu, 24 Jun 2021 22:14:51 -0700 Subject: [PATCH 3/8] update test --- .../aws-lambda-python/test/bundling.test.ts | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index cb03051eeb195..ef25190da599a 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { Code, Runtime } from '@aws-cdk/aws-lambda'; -import { FileSystem } from '@aws-cdk/core'; +import { DockerImage, FileSystem } from '@aws-cdk/core'; import { stageDependencies, bundle } from '../lib/bundling'; jest.mock('@aws-cdk/aws-lambda'); @@ -42,26 +42,6 @@ test('Bundling a function without dependencies', () => { })); }); -test('Bundling Docker with build args', () => { - const entry = path.join(__dirname, 'lambda-handler-nodeps'); - bundle({ - entry, - runtime: Runtime.PYTHON_3_7, - outputPathSuffix: '.', - buildArgs: { - HELLO: 'WORLD', - }, - }); - - expect(Code.fromAsset).toHaveBeenCalledWith(entry, - expect.objectContaining({ - buildArgs: expect.objectContaining({ - HELLO: 'WORLD', - }), - })); -}); - - test('Bundling a function with requirements.txt installed', () => { const entry = path.join(__dirname, 'lambda-handler'); bundle({ @@ -159,3 +139,22 @@ describe('Dependency detection', () => { expect(stageDependencies(sourcedir, '/dummy')).toEqual(false); }); }); + +test('Bundling Docker with build args', () => { + const entry = path.join(__dirname, 'lambda-handler-nodeps'); + bundle({ + entry, + runtime: Runtime.PYTHON_3_7, + outputPathSuffix: '.', + buildArgs: { + HELLO: 'WORLD', + }, + }); + + expect(DockerImage.fromBuild).toHaveBeenCalledWith( + expect.objectContaining({ + buildArgs: expect.objectContaining({ + HELLO: 'WORLD', + }), + })); +}); From 22fb2d27fabc08eb085f81f4c0b3786b47b04251 Mon Sep 17 00:00:00 2001 From: Setu Shah Date: Thu, 24 Jun 2021 22:22:09 -0700 Subject: [PATCH 4/8] fix test --- packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index ef25190da599a..f07424fad9de9 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -151,7 +151,7 @@ test('Bundling Docker with build args', () => { }, }); - expect(DockerImage.fromBuild).toHaveBeenCalledWith( + expect(DockerImage.fromBuild).toHaveBeenCalledWith(expect.stringMatching(/python-bundling$/), expect.objectContaining({ buildArgs: expect.objectContaining({ HELLO: 'WORLD', From 19b6e42a8bf560ea3304673e53a1914d338e603f Mon Sep 17 00:00:00 2001 From: Setu Shah Date: Thu, 24 Jun 2021 22:41:02 -0700 Subject: [PATCH 5/8] add mock for building docker images --- .../@aws-cdk/aws-lambda-python/test/bundling.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index f07424fad9de9..8dfdd346afcfb 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -5,7 +5,6 @@ import { DockerImage, FileSystem } from '@aws-cdk/core'; import { stageDependencies, bundle } from '../lib/bundling'; jest.mock('@aws-cdk/aws-lambda'); - jest.mock('child_process', () => ({ spawnSync: jest.fn(() => { return { @@ -19,8 +18,17 @@ jest.mock('child_process', () => ({ }), })); +// Mock DockerImage.fromAsset() to avoid building the image +let fromBuildMock: jest.SpyInstance; beforeEach(() => { jest.clearAllMocks(); + + fromBuildMock = jest.spyOn(DockerImage, 'fromBuild').mockReturnValue({ + image: 'built-image', + cp: () => 'dest-path', + run: () => {}, + toJSON: () => 'built-image', + }); }); test('Bundling a function without dependencies', () => { @@ -151,7 +159,7 @@ test('Bundling Docker with build args', () => { }, }); - expect(DockerImage.fromBuild).toHaveBeenCalledWith(expect.stringMatching(/python-bundling$/), + expect(fromBuildMock).toHaveBeenCalledWith(expect.stringMatching(/python-bundling$/), expect.objectContaining({ buildArgs: expect.objectContaining({ HELLO: 'WORLD', From 3916c8e7294963908a09627b5c0b4375f538489b Mon Sep 17 00:00:00 2001 From: Setu Shah Date: Thu, 24 Jun 2021 22:59:17 -0700 Subject: [PATCH 6/8] fix path --- packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index 8dfdd346afcfb..610c46c467754 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -159,7 +159,7 @@ test('Bundling Docker with build args', () => { }, }); - expect(fromBuildMock).toHaveBeenCalledWith(expect.stringMatching(/python-bundling$/), + expect(fromBuildMock).toHaveBeenCalledWith(expect.stringMatching(/\/tmp\/python-bundling$/), expect.objectContaining({ buildArgs: expect.objectContaining({ HELLO: 'WORLD', From c08d414ac45ed94cb6b361c7296feec8597e565c Mon Sep 17 00:00:00 2001 From: Setu Shah Date: Thu, 24 Jun 2021 23:14:04 -0700 Subject: [PATCH 7/8] Switch from stringMatching to stringContaining --- packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index 610c46c467754..f00510e2844bd 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -159,7 +159,7 @@ test('Bundling Docker with build args', () => { }, }); - expect(fromBuildMock).toHaveBeenCalledWith(expect.stringMatching(/\/tmp\/python-bundling$/), + expect(fromBuildMock).toHaveBeenCalledWith(expect.stringContaining('/tmp/python-bundling'), expect.objectContaining({ buildArgs: expect.objectContaining({ HELLO: 'WORLD', From 3245fd0fe1c2828fab127c2b8171f55a8930ac1a Mon Sep 17 00:00:00 2001 From: Setu Shah Date: Fri, 25 Jun 2021 11:06:41 -0700 Subject: [PATCH 8/8] revert readme lint changes --- packages/@aws-cdk/aws-lambda-python/README.md | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index f318655ca4b8b..d2428571c284d 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -28,10 +28,10 @@ Define a `PythonFunction`: import * as lambda from "@aws-cdk/aws-lambda"; import { PythonFunction } from "@aws-cdk/aws-lambda-python"; -new PythonFunction(this, "MyFunction", { - entry: "/path/to/my/function", // required - index: "my_index.py", // optional, defaults to 'index.py' - handler: "my_exported_func", // optional, defaults to 'handler' +new PythonFunction(this, 'MyFunction', { + entry: '/path/to/my/function', // required + index: 'my_index.py', // optional, defaults to 'index.py' + handler: 'my_exported_func', // optional, defaults to 'handler' runtime: lambda.Runtime.PYTHON_3_6, // optional, defaults to lambda.Runtime.PYTHON_3_7 }); ``` @@ -39,9 +39,9 @@ new PythonFunction(this, "MyFunction", { Additional `buildArgs` to use when bundling the Docker building image can be provided as: ```ts -new PythonFunction(this, "my-handler", { +new PythonFunction(this, 'my-handler', { buildArgs: { - HTTPS_PROXY: "https://127.0.0.1:3001", + HTTPS_PROXY: 'https://127.0.0.1:3001', }, }); ``` @@ -97,11 +97,11 @@ or `Pipfile` or `poetry.lock` with the associated `pyproject.toml` at the entry layer. ```ts -new lambda.PythonFunction(this, "MyFunction", { - entry: "/path/to/my/function", +new lambda.PythonFunction(this, 'MyFunction', { + entry: '/path/to/my/function', layers: [ - new lambda.PythonLayerVersion(this, "MyLayer", { - entry: "/path/to/my/layer", // point this to your library's directory + new lambda.PythonLayerVersion(this, 'MyLayer', { + entry: '/path/to/my/layer', // point this to your library's directory }), ], });