feat(core,s3-assets,lambda): custom asset bundling (#7898)

Adds support for asset bundling by running a command inside a Docker container.

The asset path is mounted in the container at `/asset-input` and is set as the working
directory. The container is responsible for putting content at `/asset-output`. The content
at `/asset-output` will be zipped and used as the final asset.

This allows to use Docker for Lambda code bundling. 

It will also be possible to refactor `aws-lambda-nodejs` and create other language
specific modules.

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk/assets/lib/api.ts

@@ -1,5 +1,7 @@
 /**
  * Common interface for all assets.
+ *
+ * @deprecated use `core.IAsset`
  */
 export interface IAsset {
   /**

packages/@aws-cdk/assets/lib/index.ts

@@ -1,4 +1,4 @@
 export * from './api';
 export * from './fs/follow-mode';
 export * from './fs/options';
-export * from './staging';
+export * from './staging';

packages/@aws-cdk/aws-lambda/README.md

@@ -32,7 +32,8 @@ runtime code.
  * `lambda.Code.fromInline(code)` - inline the handle code as a string. This is
    limited to supported runtimes and the code cannot exceed 4KiB.
  * `lambda.Code.fromAsset(path)` - specify a directory or a .zip file in the local
-   filesystem which will be zipped and uploaded to S3 before deployment.
+   filesystem which will be zipped and uploaded to S3 before deployment. See also
+   [bundling asset code](#Bundling-Asset-Code).
 
 The following example shows how to define a Python function and deploy the code
 from the local directory `my-lambda-handler` to it:
@@ -62,7 +63,7 @@ const fn = new lambda.Function(this, 'MyFunction', {
   runtime: lambda.Runtime.NODEJS_10_X,
   handler: 'index.handler',
   code: lambda.Code.fromAsset(path.join(__dirname, 'lambda-handler')),
-  
+
 fn.role // the Role
 ```
 
@@ -287,6 +288,53 @@ number of times and with different properties. Using `SingletonFunction` here wi
 For example, the `LogRetention` construct requires only one single lambda function for all different log groups whose
 retention it seeks to manage.
 
+### Bundling Asset Code
+When using `lambda.Code.fromAsset(path)` it is possible to bundle the code by running a
+command in a Docker container. The asset path will be mounted at `/asset-input`. The
+Docker container is responsible for putting content at `/asset-output`. The content at
+`/asset-output` will be zipped and used as Lambda code.
+
+Example with Python:
+```ts
+new lambda.Function(this, 'Function', {
+  code: lambda.Code.fromAsset(path.join(__dirname, 'my-python-handler'), {
+    bundling: {
+      image: lambda.Runtime.PYTHON_3_6.bundlingDockerImage,
+      command: [
+        'bash', '-c', `
+        pip install -r requirements.txt -t /asset-output &&
+        rsync -r . /asset-output
+        `,
+      ],
+    },
+  }),
+  runtime: lambda.Runtime.PYTHON_3_6,
+  handler: 'index.handler',
+});
+```
+Runtimes expose a `bundlingDockerImage` property that points to the [lambci/lambda](https://hub.docker.com/r/lambci/lambda/) build image.
+
+Use `cdk.BundlingDockerImage.fromRegistry(image)` to use an existing image or
+`cdk.BundlingDockerImage.fromAsset(path)` to build a specific image:
+
+```ts
+import * as cdk from '@aws-cdk/core';
+
+new lambda.Function(this, 'Function', {
+  code: lambda.Code.fromAsset('/path/to/handler', {
+    bundling: {
+      image: cdk.BundlingDockerImage.fromAsset('/path/to/dir/with/DockerFile', {
+        buildArgs: {
+          ARG1: 'value1',
+        },
+      }),
+      command: ['my', 'cool', 'command'],
+    },
+  }),
+  // ...
+});
+```
+
 ### Language-specific APIs
 Language-specific higher level constructs are provided in separate modules:
 

packages/@aws-cdk/aws-lambda/lib/runtime.ts

@@ -1,3 +1,5 @@
+import { BundlingDockerImage } from '@aws-cdk/core';
+
 export interface LambdaRuntimeProps {
   /**
    * Whether the ``ZipFile`` (aka inline code) property can be used with this runtime.
@@ -154,10 +156,19 @@ export class Runtime {
    */
   public readonly family?: RuntimeFamily;
 
+  /**
+   * The bundling Docker image for this runtime.
+   * Points to the lambci/lambda build image for this runtime.
+   *
+   * @see https://hub.docker.com/r/lambci/lambda/
+   */
+  public readonly bundlingDockerImage: BundlingDockerImage;
+
   constructor(name: string, family?: RuntimeFamily, props: LambdaRuntimeProps = { }) {
     this.name = name;
     this.supportsInlineCode = !!props.supportsInlineCode;
     this.family = family;
+    this.bundlingDockerImage = BundlingDockerImage.fromRegistry(`lambci/lambda:build-${name}`);
 
     Runtime.ALL.push(this);
   }

packages/@aws-cdk/aws-lambda/test/integ.bundling.expected.json

@@ -0,0 +1,113 @@
+{
+  "Resources": {
+    "FunctionServiceRole675BB04A": {
+      "Type": "AWS::IAM::Role",
+      "Properties": {
+        "AssumeRolePolicyDocument": {
+          "Statement": [
+            {
+              "Action": "sts:AssumeRole",
+              "Effect": "Allow",
+              "Principal": {
+                "Service": "lambda.amazonaws.com"
+              }
+            }
+          ],
+          "Version": "2012-10-17"
+        },
+        "ManagedPolicyArns": [
+          {
+            "Fn::Join": [
+              "",
+              [
+                "arn:",
+                {
+                  "Ref": "AWS::Partition"
+                },
+                ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+              ]
+            ]
+          }
+        ]
+      }
+    },
+    "Function76856677": {
+      "Type": "AWS::Lambda::Function",
+      "Properties": {
+        "Code": {
+          "S3Bucket": {
+            "Ref": "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3Bucket6365D8AA"
+          },
+          "S3Key": {
+            "Fn::Join": [
+              "",
+              [
+                {
+                  "Fn::Select": [
+                    0,
+                    {
+                      "Fn::Split": [
+                        "||",
+                        {
+                          "Ref": "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3VersionKey14A1DBA7"
+                        }
+                      ]
+                    }
+                  ]
+                },
+                {
+                  "Fn::Select": [
+                    1,
+                    {
+                      "Fn::Split": [
+                        "||",
+                        {
+                          "Ref": "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3VersionKey14A1DBA7"
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            ]
+          }
+        },
+        "Handler": "index.handler",
+        "Role": {
+          "Fn::GetAtt": [
+            "FunctionServiceRole675BB04A",
+            "Arn"
+          ]
+        },
+        "Runtime": "python3.6"
+      },
+      "DependsOn": [
+        "FunctionServiceRole675BB04A"
+      ]
+    }
+  },
+  "Parameters": {
+    "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3Bucket6365D8AA": {
+      "Type": "String",
+      "Description": "S3 bucket for asset \"0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cd\""
+    },
+    "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3VersionKey14A1DBA7": {
+      "Type": "String",
+      "Description": "S3 key for asset version \"0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cd\""
+    },
+    "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdArtifactHashEEC2ED67": {
+      "Type": "String",
+      "Description": "Artifact hash for asset \"0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cd\""
+    }
+  },
+  "Outputs": {
+    "FunctionArn": {
+      "Value": {
+        "Fn::GetAtt": [
+          "Function76856677",
+          "Arn"
+        ]
+      }
+    }
+  }
+}

packages/@aws-cdk/aws-lambda/test/integ.bundling.ts

@@ -0,0 +1,42 @@
+import { App, CfnOutput, Construct, Stack, StackProps } from '@aws-cdk/core';
+import * as path from 'path';
+import * as lambda from '../lib';
+
+/**
+ * Stack verification steps:
+ * * aws cloudformation describe-stacks --stack-name cdk-integ-lambda-bundling --query Stacks[0].Outputs[0].OutputValue
+ * * aws lambda invoke --function-name <output from above> response.json
+ * * cat response.json
+ * The last command should show '200'
+ */
+class TestStack extends Stack {
+  constructor(scope: Construct, id: string, props?: StackProps) {
+    super(scope, id, props);
+
+    const assetPath = path.join(__dirname, 'python-lambda-handler');
+    const fn = new lambda.Function(this, 'Function', {
+      code: lambda.Code.fromAsset(assetPath, {
+        bundling: {
+          image: lambda.Runtime.PYTHON_3_6.bundlingDockerImage,
+          command: [
+            'bash', '-c', [
+              'rsync -r . /asset-output',
+              'cd /asset-output',
+              'pip install -r requirements.txt -t .',
+            ].join(' && '),
+          ],
+        },
+      }),
+      runtime: lambda.Runtime.PYTHON_3_6,
+      handler: 'index.handler',
+    });
+
+    new CfnOutput(this, 'FunctionArn', {
+      value: fn.functionArn,
+    });
+  }
+}
+
+const app = new App();
+new TestStack(app, 'cdk-integ-lambda-bundling');
+app.synth();

packages/@aws-cdk/aws-lambda/test/python-lambda-handler/index.py

@@ -0,0 +1,8 @@
+import requests
+
+def handler(event, context):
+  r = requests.get('https://aws.amazon.com')
+
+  print(r.status_code)
+
+  return r.status_code

packages/@aws-cdk/aws-lambda/test/python-lambda-handler/requirements.txt

@@ -0,0 +1 @@
+requests==2.23.0

packages/@aws-cdk/aws-s3-assets/README.md

@@ -50,6 +50,9 @@ The following examples grants an IAM group read permissions on an asset:
 
 [Example of granting read access to an asset](./test/integ.assets.permissions.lit.ts)
 
+The following example uses custom asset bundling to convert a markdown file to html:
+[Example of using asset bundling](./test/integ.assets.bundling.lit.ts)
+
 ## How does it work?
 
 When an asset is defined in a construct, a construct metadata entry

packages/@aws-cdk/aws-s3-assets/lib/asset.ts

@@ -5,11 +5,11 @@ import * as cdk from '@aws-cdk/core';
 import * as cxapi from '@aws-cdk/cx-api';
 import * as fs from 'fs';
 import * as path from 'path';
+import { toSymlinkFollow } from './compat';
 
 const ARCHIVE_EXTENSIONS = [ '.zip', '.jar' ];
 
-export interface AssetOptions extends assets.CopyOptions {
-
+export interface AssetOptions extends assets.CopyOptions, cdk.AssetOptions {
   /**
    * A list of principals that should be able to read this asset from S3.
    * You can use `asset.grantRead(principal)` to grant read permissions later.
@@ -30,7 +30,7 @@ export interface AssetOptions extends assets.CopyOptions {
    * @default - automatically calculate source hash based on the contents
    * of the source file or directory.
    *
-   * @experimental
+   * @deprecated see `assetHash` and `assetHashType`
    */
   readonly sourceHash?: string;
 }
@@ -50,7 +50,7 @@ export interface AssetProps extends AssetOptions {
  * An asset represents a local file or directory, which is automatically uploaded to S3
  * and then can be referenced within a CDK application.
  */
-export class Asset extends cdk.Construct implements assets.IAsset {
+export class Asset extends cdk.Construct implements cdk.IAsset {
   /**
    * Attribute that represents the name of the bucket this asset exists in.
    */
@@ -98,18 +98,28 @@ export class Asset extends cdk.Construct implements assets.IAsset {
    */
   public readonly isZipArchive: boolean;
 
+  /**
+   * A cryptographic hash of the asset.
+   *
+   * @deprecated see `assetHash`
+   */
   public readonly sourceHash: string;
 
+  public readonly assetHash: string;
+
   constructor(scope: cdk.Construct, id: string, props: AssetProps) {
     super(scope, id);
 
     // stage the asset source (conditionally).
-    const staging = new assets.Staging(this, 'Stage', {
+    const staging = new cdk.AssetStaging(this, 'Stage', {
       ...props,
       sourcePath: path.resolve(props.path),
+      follow: toSymlinkFollow(props.follow),
+      assetHash: props.assetHash ?? props.sourceHash,
     });
 
-    this.sourceHash = props.sourceHash || staging.sourceHash;
+    this.assetHash = staging.assetHash;
+    this.sourceHash = this.assetHash;
 
     this.assetPath = staging.stagedPath;
 
@@ -136,7 +146,7 @@ export class Asset extends cdk.Construct implements assets.IAsset {
 
     this.bucket = s3.Bucket.fromBucketName(this, 'AssetBucket', this.s3BucketName);
 
-    for (const reader of (props.readers || [])) {
+    for (const reader of (props.readers ?? [])) {
       this.grantRead(reader);
     }
   }

packages/@aws-cdk/aws-s3-assets/lib/compat.ts

@@ -0,0 +1,17 @@
+import { FollowMode } from '@aws-cdk/assets';
+import { SymlinkFollowMode } from '@aws-cdk/core';
+
+export function toSymlinkFollow(follow?: FollowMode): SymlinkFollowMode | undefined {
+  if (!follow) {
+    return undefined;
+  }
+
+  switch (follow) {
+    case FollowMode.NEVER: return SymlinkFollowMode.NEVER;
+    case FollowMode.ALWAYS: return SymlinkFollowMode.ALWAYS;
+    case FollowMode.BLOCK_EXTERNAL: return SymlinkFollowMode.BLOCK_EXTERNAL;
+    case FollowMode.EXTERNAL: return SymlinkFollowMode.EXTERNAL;
+    default:
+      throw new Error(`unknown follow mode: ${follow}`);
+  }
+}

packages/@aws-cdk/aws-s3-assets/test/alpine-markdown/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine
+
+RUN apk add markdown

packages/@aws-cdk/aws-s3-assets/test/compat.test.ts

@@ -0,0 +1,11 @@
+import { FollowMode } from '@aws-cdk/assets';
+import { SymlinkFollowMode } from '@aws-cdk/core';
+import { toSymlinkFollow } from '../lib/compat';
+
+test('FollowMode compatibility', () => {
+  expect(toSymlinkFollow(undefined)).toBeUndefined();
+  expect(toSymlinkFollow(FollowMode.ALWAYS)).toBe(SymlinkFollowMode.ALWAYS);
+  expect(toSymlinkFollow(FollowMode.BLOCK_EXTERNAL)).toBe(SymlinkFollowMode.BLOCK_EXTERNAL);
+  expect(toSymlinkFollow(FollowMode.EXTERNAL)).toBe(SymlinkFollowMode.EXTERNAL);
+  expect(toSymlinkFollow(FollowMode.NEVER)).toBe(SymlinkFollowMode.NEVER);
+});