feat(lambda-layer-awscli): Dynamically load asset for AwsCliLayer, with bundled fallback

packages/@aws-cdk/cx-api/lib/directories.ts

@@ -0,0 +1,25 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+/**
+ * Return a location that will be used as the CDK home directory.
+ * Currently the only thing that is placed here is the cache.
+ *
+ * First try to use the users home directory (i.e. /home/someuser/),
+ * but if that directory does not exist for some reason create a tmp directory.
+ *
+ * Typically it wouldn't make sense to create a one time use tmp directory for
+ * the purpose of creating a cache, but since this only applies to users that do
+ * not have a home directory (some CI systems?) this should be fine.
+ */
+export function cdkHomeDir() {
+  const tmpDir = fs.realpathSync(os.tmpdir());
+  let home;
+  try {
+    home = path.join((os.userInfo().homedir ?? os.homedir()).trim(), '.cdk');
+  } catch {}
+  return process.env.CDK_HOME
+    ? path.resolve(process.env.CDK_HOME)
+    : home || fs.mkdtempSync(path.join(tmpDir, '.cdk')).trim();
+}

packages/@aws-cdk/cx-api/lib/index.ts

@@ -20,3 +20,4 @@ export * from './metadata';
 export * from './features';
 export * from './placeholders';
 export * from './app';
+export * from './directories';

packages/@aws-cdk/lambda-layer-awscli/README.md

@@ -25,3 +25,12 @@ fn.addLayers(new AwsCliLayer(this, 'AwsCliLayer'));
 ```
 
 The CLI will be installed under `/opt/awscli/aws`.
+
+## Troubleshooting
+
+### WARNING! ACTION REQUIRED! Your CDK application is using AwsCliLayer and this construct may experience a breaking change for your environment in a future release.
+
+If you see the above message when synthesizing your CDK app, this is because we have introduced a change to dynamically load the Asset construct used by AwsCliLayer at runtime. This message appears when the dynamic loading fails due to restrictions in your environment, and it falls back to using an Asset bundled in aws-cdk-lib. We plan to remove this fallback, and at that time your application may be broken. To prevent this, add an explicit dependency on @aws-cdk/asset-awscli-v1. If you experience any issues, please reach out to us by commenting on https://github.com/aws/aws-cdk/issues/999999999.
+
+TODO:
+Add language-specific instructions.

packages/@aws-cdk/lambda-layer-awscli/jest.config.js

@@ -1,2 +1,9 @@
 const baseConfig = require('@aws-cdk/cdk-build-tools/config/jest.config');
-module.exports = baseConfig;
+module.exports = {
+  ...baseConfig,
+  coverageThreshold: {
+    global: {
+      branches: 60,
+    },
+  },
+};

packages/@aws-cdk/lambda-layer-awscli/layer/download-fallback.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Get the version to use from the package.json devDependencies
+lineWithPackageVersion=$(grep '@aws-cdk/asset-awscli-v1' ./package.json)
+version=$(echo $lineWithPackageVersion | cut -d '"' -f 4)
+
+echo "Downloading @aws-cdk/asset-awscli-v1@$version from npm"
+npm pack @aws-cdk/asset-awscli-v1@$version -q
+
+echo "Extracting layer.zip and requirements.txt from aws-cdk-asset-awscli-v1-$version.tgz"
+tar -zxvf aws-cdk-asset-awscli-v1-$version.tgz package/lib/layer.zip package/layer/requirements.txt
+
+echo "Moving layer.zip and requirements.txt to desired destinations"
+mv ./package/lib/layer.zip ./lib/
+mv ./package/layer/requirements.txt ./layer
+
+echo "Cleaning up"
+rm aws-cdk-asset-awscli-v1-$version.tgz
+rm -r ./package
+
+echo "download-fallback.sh complete"

packages/@aws-cdk/lambda-layer-awscli/layer/requirements.txt

@@ -1 +1 @@
-awscli==1.25.70
+awscli==1.25.46

packages/@aws-cdk/lambda-layer-awscli/lib/awscli-layer.ts

@@ -1,19 +1,150 @@
+/* eslint-disable no-console */
+import * as childproc from 'child_process';
+import * as fs from 'fs';
 import * as path from 'path';
 import * as lambda from '@aws-cdk/aws-lambda';
-import { FileSystem } from '@aws-cdk/core';
+import { Annotations, FileSystem } from '@aws-cdk/core';
+import { debugModeEnabled } from '@aws-cdk/core/lib/debug';
+import * as cxapi from '@aws-cdk/cx-api';
 import { Construct } from 'constructs';
 
 /**
  * An AWS Lambda layer that includes the AWS CLI.
  */
 export class AwsCliLayer extends lambda.LayerVersion {
+  /**
+   * @internal
+   */
+  public static _tryLoadPackage(targetVersion: string, logs: string[]): any {
+    let availableVersion;
+    let assetPackagePath;
+    try {
+      assetPackagePath = require.resolve(`${AwsCliLayer.assetPackageName}`);
+    } catch (err) {
+      logs.push(`require.resolve('${AwsCliLayer.assetPackageName}') failed`);
+      return;
+    }
+    availableVersion = AwsCliLayer.requireWrapper(path.join(assetPackagePath, '../../package.json'), logs).version;
+
+    if (targetVersion === availableVersion) {
+      logs.push(`${AwsCliLayer.assetPackageName} already installed with correct version: ${availableVersion}.`);
+      const result = AwsCliLayer.requireWrapper(AwsCliLayer.assetPackageName, logs);
+      if (result) {
+        logs.push(`Successfully loaded ${AwsCliLayer.assetPackageName} from pre-installed packages.`);
+        return result;
+      }
+    }
+  }
+
+  /**
+   * @internal
+   */
+  public static _downloadPackage(targetVersion: string, logs: string[]): string | undefined {
+    const cdkHomeDir = cxapi.cdkHomeDir();
+    const downloadDir = path.join(cdkHomeDir, 'npm-cache');
+    const downloadPath = path.join(downloadDir, `${AwsCliLayer.assetPackageNpmTarPrefix}${targetVersion}.tgz`);
+
+    if (fs.existsSync(downloadPath)) {
+      logs.push(`Using package archive already available at location: ${downloadPath}`);
+      return downloadPath;
+    }
+    logs.push(`Downloading package using npm pack to: ${downloadDir}`);
+    childproc.execSync(`mkdir -p ${downloadDir}; cd ${downloadDir}; npm pack ${AwsCliLayer.assetPackageName}@${targetVersion} -q`);
+    if (fs.existsSync(downloadPath)) {
+      logs.push('Successfully downloaded using npm pack.');
+      return downloadPath;
+    }
+
+    logs.push('Failed to download using npm pack.');
+    return undefined;
+  }
+
+  private static readonly assetPackageName: string = '@aws-cdk/asset-awscli-v1';
+  private static readonly assetPackageNpmTarPrefix: string = 'aws-cdk-asset-awscli-v1-';
+
+  private static requireWrapper(id: string, logs: string[]): any {
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      const result = require(id);
+      if (result) {
+        logs.push(`require('${id}') succeeded.`);
+      }
+      return result;
+    } catch (e) {
+      logs.push(`require('${id}') failed.`);
+      const eAsError = e as Error;
+      if (eAsError?.stack) {
+        logs.push(eAsError.stack);
+      }
+    }
+  }
+
+  private static installAndLoadPackage(from: string, logs: string[]): any {
+    const installDir = AwsCliLayer.findInstallDir();
+    if (!installDir) {
+      logs.push('Unable to find an install directory. require.main.paths[0] is undefined.');
+      return;
+    }
+    logs.push(`Installing from: ${from} to: ${installDir}`);
+    childproc.execSync(`npm install ${from} --no-save --prefix ${installDir} -q`);
+    return AwsCliLayer.requireWrapper(path.join(installDir, 'node_modules', AwsCliLayer.assetPackageName, 'lib/index.js'), logs);
+  }
+
+  private static findInstallDir(): string | undefined {
+    if (!require.main?.paths) {
+      return undefined;
+    }
+    return require.main.paths[0];
+  }
+
   constructor(scope: Construct, id: string) {
-    super(scope, id, {
-      code: lambda.Code.fromAsset(path.join(__dirname, 'layer.zip'), {
+    const logs: string[] = [];
+    let fallback = false;
+
+    const targetVersion = AwsCliLayer.requireWrapper(path.join(__dirname, '../package.json'), logs).devDependencies[AwsCliLayer.assetPackageName];
+
+    let assetPackage = AwsCliLayer._tryLoadPackage(targetVersion, logs);
+
+    if (!assetPackage) {
+      const downloadPath = AwsCliLayer._downloadPackage(targetVersion, logs);
+      if (downloadPath) {
+        assetPackage = AwsCliLayer.installAndLoadPackage(downloadPath, logs);
+      }
+    }
+
+    let code;
+    if (assetPackage) {
+      // ask for feedback here
+      const asset = new assetPackage.AwsCliAsset(scope, `${id}-asset`);
+      code = lambda.Code.fromBucket(asset.bucket, asset.s3ObjectKey);
+    }
+
+    if (!code) {
+      fallback = true;
+      logs.push(`Unable to load ${AwsCliLayer.assetPackageName}. Falling back to use layer.zip bundled with aws-cdk-lib`);
+      code = lambda.Code.fromAsset(path.join(__dirname, 'layer.zip'), {
         // we hash the layer directory (it contains the tools versions and Dockerfile) because hashing the zip is non-deterministic
         assetHash: FileSystem.fingerprint(path.join(__dirname, '../layer')),
-      }),
+      });
+    }
+
+    super(scope, id, {
+      code: code,
       description: '/opt/awscli/aws',
     });
+
+    if (debugModeEnabled()) {
+      Annotations.of(this).addInfo(logs.join('\n'));
+    }
+
+    if (fallback) {
+      Annotations.of(this).addWarning(`WARNING! ACTION REQUIRED! Your CDK application is using ${this.constructor.name} and this construct may experience a breaking change for your environment in a future release. See https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.lambda_layer_awscli-readme.html or https://github.com/aws/aws-cdk/issues/999999999 for details and resolution instructions.`);
+      new Notice(this, 'cli-notice');
+    }
   }
 }
+
+/**
+ * An empty construct that can be added to the tree as a marker for the CLI Notices
+ */
+class Notice extends Construct {}

packages/@aws-cdk/lambda-layer-awscli/package.json

@@ -78,18 +78,23 @@
     "@aws-cdk/integ-runner": "0.0.0",
     "@aws-cdk/integ-tests": "0.0.0",
     "@aws-cdk/pkglint": "0.0.0",
+    "@aws-cdk/asset-awscli-v1": "0.0.13",
     "@types/jest": "^27.5.2",
     "jest": "^27.5.1"
   },
   "dependencies": {
     "@aws-cdk/aws-lambda": "0.0.0",
+    "@aws-cdk/aws-s3-assets": "0.0.0",
     "@aws-cdk/core": "0.0.0",
+    "@aws-cdk/cx-api": "0.0.0",
     "constructs": "^10.0.0"
   },
   "homepage": "https://github.com/aws/aws-cdk",
   "peerDependencies": {
     "@aws-cdk/aws-lambda": "0.0.0",
+    "@aws-cdk/aws-s3-assets": "0.0.0",
     "@aws-cdk/core": "0.0.0",
+    "@aws-cdk/cx-api": "0.0.0",
     "constructs": "^10.0.0"
   },
   "engines": {
@@ -102,7 +107,7 @@
   },
   "cdk-build": {
     "pre": [
-      "layer/build.sh"
+      "layer/download-fallback.sh"
     ],
     "env": {
       "AWSLINT_BASE_CONSTRUCT": true
@@ -120,5 +125,10 @@
   "publishConfig": {
     "tag": "latest"
   },
+  "awslint": {
+    "exclude": [
+      "props-physical-name:@aws-cdk/lambda-layer-awscli.AwsCliLayerProps"
+    ]
+  },
   "private": true
 }