feat(lambda-layer-awscli): Dynamically load asset for AwsCliLayer, with bundled fallback

package.json

@@ -115,6 +115,8 @@
       "@aws-cdk/core/minimatch/**",
       "@aws-cdk/cx-api/semver",
       "@aws-cdk/cx-api/semver/**",
+      "@aws-cdk/lambda-layer-awscli/semver",
+      "@aws-cdk/lambda-layer-awscli/semver/**",
       "@aws-cdk/pipelines/aws-sdk",
       "@aws-cdk/pipelines/aws-sdk/**",
       "@aws-cdk/yaml-cfn/yaml",

packages/@aws-cdk/cx-api/jest.config.js

@@ -4,7 +4,7 @@ module.exports = {
     coverageThreshold: {
         global: {
             ...baseConfig.coverageThreshold.global,
-            branches: 75,
+            branches: 70,
         },
     },
 };

packages/@aws-cdk/cx-api/lib/directories.ts

@@ -0,0 +1,25 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+/**
+ * Return a location that will be used as the CDK home directory.
+ * Currently the only thing that is placed here is the cache.
+ *
+ * First try to use the users home directory (i.e. /home/someuser/),
+ * but if that directory does not exist for some reason create a tmp directory.
+ *
+ * Typically it wouldn't make sense to create a one time use tmp directory for
+ * the purpose of creating a cache, but since this only applies to users that do
+ * not have a home directory (some CI systems?) this should be fine.
+ */
+export function cdkHomeDir() {
+  const tmpDir = fs.realpathSync(os.tmpdir());
+  let home;
+  try {
+    home = path.join((os.userInfo().homedir ?? os.homedir()).trim(), '.cdk');
+  } catch {}
+  return process.env.CDK_HOME
+    ? path.resolve(process.env.CDK_HOME)
+    : home || fs.mkdtempSync(path.join(tmpDir, '.cdk')).trim();
+}

packages/@aws-cdk/cx-api/lib/index.ts

@@ -20,3 +20,4 @@ export * from './metadata';
 export * from './features';
 export * from './placeholders';
 export * from './app';
+export * from './directories';

packages/@aws-cdk/lambda-layer-awscli/.gitignore

@@ -18,6 +18,7 @@ nyc.config.js
 
 junit.xml
 lib/*.zip
+layer/requirements.txt
 
 !**/*.integ.snapshot/**/asset.*/*.js
 !**/*.integ.snapshot/**/asset.*/*.d.ts

packages/@aws-cdk/lambda-layer-awscli/README.md

@@ -25,3 +25,20 @@ fn.addLayers(new AwsCliLayer(this, 'AwsCliLayer'));
 ```
 
 The CLI will be installed under `/opt/awscli/aws`.
+
+## Troubleshooting
+
+### WARNING! [ACTION REQUIRED] Your CDK application is using ${this.constructor.name}. Add a dependency on ${AwsCliLayer.assetPackageName}, or the equivalent in your language, to remove this warning
+
+If you see the above message when synthesizing your CDK app, this is because we
+have introduced a change to dynamically load the Asset construct used by
+AwsCliLayer at runtime. This message appears when the dynamic loading fails due
+to restrictions in your environment, and it falls back to using an Asset bundled
+in aws-cdk-lib. We plan to remove this fallback, and at that time your
+application may be broken. To prevent this, add an explicit dependency on
+@aws-cdk/asset-awscli-v1, or the equivalent in your language. If you experience
+any issues, please reach out to us by commenting on
+https://github.com/aws/aws-cdk/issues/22470.
+
+TODO:
+Add language-specific instructions.

packages/@aws-cdk/lambda-layer-awscli/jest.config.js

@@ -1,2 +1,9 @@
 const baseConfig = require('@aws-cdk/cdk-build-tools/config/jest.config');
-module.exports = baseConfig;
+module.exports = {
+  ...baseConfig,
+  coverageThreshold: {
+    global: {
+      branches: 45,
+    },
+  },
+};

packages/@aws-cdk/lambda-layer-awscli/layer/pre-build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Copy files from the @aws-cdk/asset-awscli-v1 devDependency into
+# this package, so that they will be included in the released package.
+
+# Set bash to exit the script immediately on any error (e) and if any unset (u)
+# variable is referenced.
+set -eu
+
+dir=$(node -pe 'path.resolve(require.resolve("@aws-cdk/asset-awscli-v1"), "../..")')
+
+cp $dir/layer/requirements.txt ./layer
+cp $dir/lib/layer.zip ./lib
+
+target_version=$(node -pe 'require("@aws-cdk/asset-awscli-v1/package.json").version')
+echo "export const TARGET_VERSION = '${target_version}';" > lib/asset-package-version.ts

packages/@aws-cdk/lambda-layer-awscli/lib/asset-package-version.ts

@@ -0,0 +1 @@
+export const TARGET_VERSION = '2.1.0';

packages/@aws-cdk/lambda-layer-awscli/lib/awscli-layer.ts

@@ -1,19 +1,70 @@
 import * as path from 'path';
 import * as lambda from '@aws-cdk/aws-lambda';
-import { FileSystem } from '@aws-cdk/core';
+import { Annotations, FileSystem } from '@aws-cdk/core';
+import { debugModeEnabled } from '@aws-cdk/core/lib/debug';
 import { Construct } from 'constructs';
+import * as semver from 'semver';
+import { TARGET_VERSION } from './asset-package-version';
+import { installAndLoadPackage, _downloadPackage, _tryLoadPackage } from './private/package-loading-functions';
 
 /**
  * An AWS Lambda layer that includes the AWS CLI.
  */
 export class AwsCliLayer extends lambda.LayerVersion {
+
+  private static readonly assetPackageName: string = '@aws-cdk/asset-awscli-v1';
+  private static readonly assetPackageNpmTarPrefix: string = 'aws-cdk-asset-awscli-v1-';
+
   constructor(scope: Construct, id: string) {
-    super(scope, id, {
-      code: lambda.Code.fromAsset(path.join(__dirname, 'layer.zip'), {
+    const logs: string[] = [];
+    let fallback = false;
+
+    let assetPackage = _tryLoadPackage(AwsCliLayer.assetPackageName, new semver.Range(`^${TARGET_VERSION}`), logs);
+
+    if (!assetPackage) {
+      const downloadPath = _downloadPackage(AwsCliLayer.assetPackageName, AwsCliLayer.assetPackageNpmTarPrefix, TARGET_VERSION, logs);
+      if (downloadPath) {
+        assetPackage = installAndLoadPackage(AwsCliLayer.assetPackageName, downloadPath, logs);
+      }
+    }
+
+    let code;
+    if (assetPackage) {
+      // ask for feedback here
+      if (!assetPackage.AwsCliAsset) {
+        logs.push(`ERROR: loaded ${AwsCliLayer.assetPackageName}, but AwsCliAsset is undefined in the module.`);
+      } else {
+        const asset = new assetPackage.AwsCliAsset(scope, `${id}-asset`);
+        code = lambda.Code.fromBucket(asset.bucket, asset.s3ObjectKey);
+      }
+    }
+
+    if (!code) {
+      fallback = true;
+      logs.push(`Unable to load ${AwsCliLayer.assetPackageName}. Falling back to use layer.zip bundled with aws-cdk-lib`);
+      code = lambda.Code.fromAsset(path.join(__dirname, 'layer.zip'), {
         // we hash the layer directory (it contains the tools versions and Dockerfile) because hashing the zip is non-deterministic
         assetHash: FileSystem.fingerprint(path.join(__dirname, '../layer')),
-      }),
+      });
+    }
+
+    super(scope, id, {
+      code: code,
       description: '/opt/awscli/aws',
     });
+
+    if (debugModeEnabled()) {
+      Annotations.of(this).addInfo(logs.join('\n'));
+    }
+
+    if (fallback) {
+      Annotations.of(this).addWarning(`[ACTION REQUIRED] Your CDK application is using ${this.constructor.name}. Add a dependency on ${AwsCliLayer.assetPackageName}, or the equivalent in your language, to remove this warning. See https://github.com/aws/aws-cdk/issues/22470 for more information.`);
+      new Notice(this, 'cli-notice');
+    }
   }
 }
+
+/**
+ * An empty construct that can be added to the tree as a marker for the CLI Notices
+ */
+class Notice extends Construct {}

packages/@aws-cdk/lambda-layer-awscli/lib/private/package-loading-functions.ts

@@ -0,0 +1,116 @@
+import * as childproc from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as cxapi from '@aws-cdk/cx-api';
+import * as semver from 'semver';
+
+export function _tryLoadPackage(packageName: string, targetVersionRange: semver.Range, logs: string[]): any {
+  let availableVersion;
+  let assetPackagePath;
+  try {
+    assetPackagePath = require.resolve(`${packageName}`);
+  } catch (e) {
+    logs.push(`require.resolve('${packageName}') failed`);
+    const eAsError = e as Error;
+    if (eAsError?.stack) {
+      logs.push(eAsError.stack);
+    }
+    return;
+  }
+  availableVersion = requireWrapper(path.join(assetPackagePath, '../../package.json'), logs).version;
+
+  if (semver.satisfies(availableVersion, targetVersionRange)) {
+    logs.push(`${packageName} already installed with compatible version: ${availableVersion}.`);
+    const result = requireWrapper(packageName, logs);
+    if (result) {
+      logs.push(`Successfully loaded ${packageName} from pre-installed packages.`);
+      return result;
+    }
+  } else {
+    logs.push(`${packageName} already installed with incompatible version: ${availableVersion}. Target version range was: ${targetVersionRange}.`);
+  }
+}
+
+export function _downloadPackage(packageName: string, packageNpmTarPrefix: string, targetVersion: string, logs: string[]): string | undefined {
+  const cdkHomeDir = cxapi.cdkHomeDir();
+  const downloadDir = path.join(cdkHomeDir, 'npm-cache');
+  const downloadPath = path.join(downloadDir, `${packageNpmTarPrefix}${targetVersion}.tgz`);
+
+  if (fs.existsSync(downloadPath)) {
+    logs.push(`Using package archive already available at location: ${downloadPath}`);
+    return downloadPath;
+  }
+
+  logs.push(`Creating directory: ${downloadDir}`);
+  try {
+    fs.mkdirSync(downloadDir);
+  } catch (e) {
+    if ((e as any)?.code === 'EEXIST') {
+      logs.push(`Directory ${downloadDir} already exists.`);
+    } else {
+      logs.push('mkdirSync() failed');
+      const eAsError = e as Error;
+      if (eAsError.stack) {
+        logs.push(eAsError.stack);
+      }
+      return undefined;
+    }
+  }
+
+  try {
+    childproc.execSync(`npm pack ${packageName}@${targetVersion} -q`, {
+      cwd: downloadDir,
+    });
+  } catch (e) {
+    logs.push('npm pack failed or timed out');
+    const eAsError = e as Error;
+    logs.push(eAsError.name);
+    logs.push(eAsError.message);
+    if (eAsError.stack) {
+      logs.push(eAsError.stack);
+    }
+  }
+
+  if (fs.existsSync(downloadPath)) {
+    logs.push('Successfully downloaded using npm pack.');
+    return downloadPath;
+  }
+
+  logs.push('Failed to download using npm pack.');
+  return undefined;
+}
+
+export function requireWrapper(id: string, logs: string[]): any {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const result = require(id);
+    if (result) {
+      logs.push(`require('${id}') succeeded.`);
+    }
+    return result;
+  } catch (e) {
+    logs.push(`require('${id}') failed.`);
+    const eAsError = e as Error;
+    if (eAsError?.stack) {
+      logs.push(eAsError.stack);
+    }
+  }
+}
+
+export function installAndLoadPackage(packageName: string, from: string, logs: string[]): any {
+  const installDir = findInstallDir();
+  if (!installDir) {
+    logs.push('Unable to find an install directory. require.main.paths[0] is undefined.');
+    return;
+  }
+  logs.push(`Installing from: ${from} to: ${installDir}`);
+  childproc.execSync(`npm install ${from} --no-save --prefix ${installDir} -q`);
+  return requireWrapper(path.join(installDir, 'node_modules', packageName, 'lib/index.js'), logs);
+}
+
+export function findInstallDir(): string | undefined {
+  if (!require.main?.paths) {
+    return undefined;
+  }
+  return path.dirname(require.main.paths[0]);
+}