feat(core,s3-assets): custom bundling docker command

In order to support environments in which docker cannot be executed or has a unique location, we added an environment variable `CDK_DOCKER` which is used instead of `docker` if defined.

Resolves #8460

packages/@aws-cdk/aws-s3-assets/README.md

@@ -50,9 +50,6 @@ The following examples grants an IAM group read permissions on an asset:
 
 [Example of granting read access to an asset](./test/integ.assets.permissions.lit.ts)
 
-The following example uses custom asset bundling to convert a markdown file to html:
-[Example of using asset bundling](./test/integ.assets.bundling.lit.ts)
-
 ## How does it work?
 
 When an asset is defined in a construct, a construct metadata entry
@@ -73,6 +70,26 @@ the asset store, it is uploaded during deployment.
 Now, when the toolkit deploys the stack, it will set the relevant CloudFormation
 Parameters to point to the actual bucket and key for each asset.
 
+## Asset Bundling
+
+When defining an asset, you can use the `bundling` option to specify a command
+to run inside a docker container. The command can read the contents of the asset
+source from `/asset-input` and is expected to write files under `/asset-output`
+(directories mapped inside the container). The files under `/asset-output` will
+be zipped and uploaded to S3 as the asset.
+
+The following example uses custom asset bundling to convert a markdown file to html:
+
+[Example of using asset bundling](./test/integ.assets.bundling.lit.ts).
+
+The bundling docker image (`image`) can either come from a registry (`BundlingDockerImage.fromRegistry`)
+or it can be built from a `Dockerfile` located inside your project (`BundlingDockerImage.fromAsset`).
+
+You can set the `CDK_DOCKER` environment variable in order to provide a custom
+docker program to execute. This may sometime be needed when building in
+environments where the standard docker cannot be executed (see
+https://github.com/aws/aws-cdk/issues/8460 for details).
+
 ## CloudFormation Resource Metadata
 
 > NOTE: This section is relevant for authors of AWS Resource Constructs.

packages/@aws-cdk/core/lib/asset-staging.ts

@@ -3,7 +3,7 @@ import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 import { AssetHashType, AssetOptions } from './assets';
-import { BUNDLING_INPUT_DIR, BUNDLING_OUTPUT_DIR, BundlingOptions } from './bundling';
+import { BundlingOptions } from './bundling';
 import { Construct, ISynthesisSession } from './construct-compat';
 import { FileSystem, FingerprintOptions } from './fs';
 
@@ -36,6 +36,18 @@ export interface AssetStagingProps extends FingerprintOptions, AssetOptions {
  * means that only if content was changed, copy will happen.
  */
 export class AssetStaging extends Construct {
+  /**
+   * The directory inside the bundling container into the asset sources will be mounted.
+   * @experimental
+   */
+  public static readonly BUNDLING_INPUT_DIR = '/asset-input';
+
+  /**
+   * The directory inside the bundling container into which the bundled output should be written.
+   * @experimental
+   */
+  public static readonly BUNDLING_OUTPUT_DIR = '/asset-output';
+
   /**
    * The path to the asset (stringinfied token).
    *
@@ -143,11 +155,11 @@ export class AssetStaging extends Construct {
     const volumes = [
       {
         hostPath: this.sourcePath,
-        containerPath: BUNDLING_INPUT_DIR,
+        containerPath: AssetStaging.BUNDLING_INPUT_DIR,
       },
       {
         hostPath: bundleDir,
-        containerPath: BUNDLING_OUTPUT_DIR,
+        containerPath: AssetStaging.BUNDLING_OUTPUT_DIR,
       },
       ...options.volumes ?? [],
     ];
@@ -157,14 +169,14 @@ export class AssetStaging extends Construct {
         command: options.command,
         volumes,
         environment: options.environment,
-        workingDirectory: options.workingDirectory ?? BUNDLING_INPUT_DIR,
+        workingDirectory: options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR,
       });
     } catch (err) {
       throw new Error(`Failed to run bundling Docker image for asset ${this.node.path}: ${err}`);
     }
 
     if (FileSystem.isEmpty(bundleDir)) {
-      throw new Error(`Bundling did not produce any output. Check that your container writes content to ${BUNDLING_OUTPUT_DIR}.`);
+      throw new Error(`Bundling did not produce any output. Check that your container writes content to ${AssetStaging.BUNDLING_OUTPUT_DIR}.`);
     }
 
     return bundleDir;

packages/@aws-cdk/core/lib/bundling.ts

@@ -1,8 +1,5 @@
 import { spawnSync } from 'child_process';
 
-export const BUNDLING_INPUT_DIR = '/asset-input';
-export const BUNDLING_OUTPUT_DIR = '/asset-output';
-
 /**
  * Bundling options
  *
@@ -75,7 +72,7 @@ export class BundlingDockerImage {
       path,
     ];
 
-    const docker = exec('docker', dockerArgs);
+    const docker = dockerExec(dockerArgs);
 
     const match = docker.stdout.toString().match(/Successfully built ([a-z0-9]+)/);
 
@@ -110,7 +107,7 @@ export class BundlingDockerImage {
       ...command,
     ];
 
-    exec('docker', dockerArgs);
+    dockerExec(dockerArgs);
   }
 }
 
@@ -178,8 +175,9 @@ function flatten(x: string[][]) {
   return Array.prototype.concat([], ...x);
 }
 
-function exec(cmd: string, args: string[]) {
-  const proc = spawnSync(cmd, args);
+function dockerExec(args: string[]) {
+  const prog = process.env.CDK_DOCKER ?? 'docker';
+  const proc = spawnSync(prog, args);
 
   if (proc.error) {
     throw proc.error;

packages/@aws-cdk/core/test/docker-stub.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -euo pipefail
+
+# stub for the `docker` executable. it is used as CDK_DOCKER when executing unit
+# tests in `test.staging.ts` It outputs the command line to
+# `/tmp/docker-stub.input` and accepts one of 3 commands that impact it's
+# behavior.
+
+echo "$@" > /tmp/docker-stub.input
+
+if echo "$@" | grep "DOCKER_STUB_SUCCESS_NO_OUTPUT"; then
+  exit 0
+fi
+
+if echo "$@" | grep "DOCKER_STUB_FAIL"; then
+  echo "A HUGE FAILING DOCKER STUFF"
+  exit 1
+fi
+
+if echo "$@" | grep "DOCKER_STUB_SUCCESS"; then
+  outdir=$(echo "$@" | xargs -n1 | grep "/asset-output" | head -n1 | cut -d":" -f1)
+  touch ${outdir}/test.txt
+  exit 0
+fi
+
+echo "Docker mock only supports one of the following commands: DOCKER_STUB_SUCCESS_NO_OUTPUT,DOCKER_STUB_FAIL,DOCKER_STUB_SUCCESS"
+exit 1

packages/@aws-cdk/core/test/test.staging.ts

@@ -4,6 +4,15 @@ import { Test } from 'nodeunit';
 import * as path from 'path';
 import { App, AssetHashType, AssetStaging, BundlingDockerImage, Stack } from '../lib';
 
+enum DockerStubCommand {
+  SUCCESS           = 'DOCKER_STUB_SUCCESS',
+  FAIL              = 'DOCKER_STUB_FAIL',
+  SUCCESS_NO_OUTPUT = 'DOCKER_STUB_SUCCESS_NO_OUTPUT'
+}
+
+// this is a way to provide a custom "docker" command for staging.
+process.env.CDK_DOCKER = `${__dirname}/docker-stub.sh`;
+
 export = {
   'base case'(test: Test) {
     // GIVEN
@@ -86,10 +95,12 @@ export = {
       sourcePath: directory,
       bundling: {
         image: BundlingDockerImage.fromRegistry('alpine'),
-        command: ['touch', '/asset-output/test.txt'],
+        command: [ DockerStubCommand.SUCCESS ],
       },
     });
 
+    test.deepEqual(readDockerStubInput(), 'run --rm -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS');
+
     // THEN
     const assembly = app.synth();
     test.deepEqual(fs.readdirSync(assembly.directory), [
@@ -114,9 +125,12 @@ export = {
       sourcePath: directory,
       bundling: {
         image: BundlingDockerImage.fromRegistry('alpine'),
+        command: [ DockerStubCommand.SUCCESS_NO_OUTPUT ],
       },
     }), /Bundling did not produce any output/);
 
+    test.equal(readDockerStubInput(),
+      'run --rm -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS_NO_OUTPUT');
     test.done();
   },
 
@@ -131,11 +145,12 @@ export = {
       sourcePath: directory,
       bundling: {
         image: BundlingDockerImage.fromRegistry('alpine'),
-        command: ['touch', '/asset-output/test.txt'],
+        command: [ DockerStubCommand.SUCCESS ],
       },
       assetHashType: AssetHashType.BUNDLE,
     });
 
+    test.equal(readDockerStubInput(), 'run --rm -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS');
     test.equal(asset.assetHash, '33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f');
 
     test.done();
@@ -153,6 +168,7 @@ export = {
       assetHash: 'my-custom-hash',
     });
 
+    test.equal(readDockerStubInput(), 'run --rm -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS');
     test.equal(asset.assetHash, 'my-custom-hash');
 
     test.done();
@@ -169,11 +185,12 @@ export = {
       sourcePath: directory,
       bundling: {
         image: BundlingDockerImage.fromRegistry('alpine'),
-        command: ['touch', '/asset-output/test.txt'],
+        command: [ DockerStubCommand.SUCCESS ],
       },
       assetHash: 'my-custom-hash',
       assetHashType: AssetHashType.BUNDLE,
     }), /Cannot specify `bundle` for `assetHashType`/);
+    test.equal(readDockerStubInput(), 'run --rm -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS');
 
     test.done();
   },
@@ -189,6 +206,7 @@ export = {
       sourcePath: directory,
       assetHashType: AssetHashType.BUNDLE,
     }), /Cannot use `AssetHashType.BUNDLE` when `bundling` is not specified/);
+    test.equal(readDockerStubInput(), 'run --rm -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS');
 
     test.done();
   },
@@ -204,6 +222,7 @@ export = {
       sourcePath: directory,
       assetHashType: AssetHashType.CUSTOM,
     }), /`assetHash` must be specified when `assetHashType` is set to `AssetHashType.CUSTOM`/);
+    test.equal(readDockerStubInput(), 'run --rm -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS');
 
     test.done();
   },
@@ -219,9 +238,18 @@ export = {
       sourcePath: directory,
       bundling: {
         image: BundlingDockerImage.fromRegistry('this-is-an-invalid-docker-image'),
+        command: [ DockerStubCommand.FAIL ],
       },
     }), /Failed to run bundling Docker image for asset stack\/Asset/);
+    test.equal(readDockerStubInput(), 'run --rm -v /input:/asset-input -v /output:/asset-output -w /asset-input this-is-an-invalid-docker-image DOCKER_STUB_FAIL');
 
     test.done();
   },
 };
+
+function readDockerStubInput() {
+  const out = fs.readFileSync('/tmp/docker-stub.input', 'utf-8').trim();
+  return out
+    .replace(/([\/a-zA-Z0-9-@_]+):\/asset-input/, '/input:/asset-input')
+    .replace(/([\/a-zA-Z0-9-@_]+):\/asset-output/, '/output:/asset-output');
+}