feat(lambda-layer-awscli): Dynamically load asset for AwsCliLayer, with bundled fallback

packages/@aws-cdk/cx-api/lib/directories.ts

@@ -0,0 +1,25 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+/**
+ * Return a location that will be used as the CDK home directory.
+ * Currently the only thing that is placed here is the cache.
+ *
+ * First try to use the users home directory (i.e. /home/someuser/),
+ * but if that directory does not exist for some reason create a tmp directory.
+ *
+ * Typically it wouldn't make sense to create a one time use tmp directory for
+ * the purpose of creating a cache, but since this only applies to users that do
+ * not have a home directory (some CI systems?) this should be fine.
+ */
+export function cdkHomeDir() {
+  const tmpDir = fs.realpathSync(os.tmpdir());
+  let home;
+  try {
+    home = path.join((os.userInfo().homedir ?? os.homedir()).trim(), '.cdk');
+  } catch {}
+  return process.env.CDK_HOME
+    ? path.resolve(process.env.CDK_HOME)
+    : home || fs.mkdtempSync(path.join(tmpDir, '.cdk')).trim();
+}

packages/@aws-cdk/cx-api/lib/index.ts

@@ -20,3 +20,4 @@ export * from './metadata';
 export * from './features';
 export * from './placeholders';
 export * from './app';
+export * from './directories';

packages/@aws-cdk/lambda-layer-awscli/jest.config.js

@@ -1,2 +1,9 @@
 const baseConfig = require('@aws-cdk/cdk-build-tools/config/jest.config');
-module.exports = baseConfig;
+module.exports = {
+  ...baseConfig,
+  coverageThreshold: {
+    global: {
+      branches: 60,
+    },
+  },
+};

packages/@aws-cdk/lambda-layer-awscli/layer/download-fallback.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Get the version to use from the package.json devDependencies
+lineWithPackageVersion=$(grep '@aws-cdk/asset-awscli-v1' ./package.json)
+version=$(echo $lineWithPackageVersion | cut -d '"' -f 4)
+
+echo "Downloading @aws-cdk/asset-awscli-v1@$version from npm"
+npm pack @aws-cdk/asset-awscli-v1@$version -q
+
+echo "Extracting layer.zip from aws-cdk-asset-awscli-v1-$version.tgz"
+tar -zxvf aws-cdk-asset-awscli-v1-$version.tgz package/lib/layer.zip
+
+echo "Copying layer.zip to ./layer"
+mv ./package/lib/layer.zip ./lib/
+
+echo "Cleaning up"
+rm aws-cdk-asset-awscli-v1-$version.tgz
+rm -r ./package
+
+echo "download-fallback.sh complete"

packages/@aws-cdk/lambda-layer-awscli/lib/awscli-layer.ts

@@ -1,19 +1,150 @@
+/* eslint-disable no-console */
+import * as childproc from 'child_process';
+import * as fs from 'fs';
 import * as path from 'path';
 import * as lambda from '@aws-cdk/aws-lambda';
 import { FileSystem } from '@aws-cdk/core';
+import * as cxapi from '@aws-cdk/cx-api';
 import { Construct } from 'constructs';
 
+/**
+ * Options for AwsCliLayer
+ */
+export interface AwsCliLayerProps {
+  /**
+   * Filter out logging statements.
+   *
+   * @default true
+   */
+  readonly quiet?: boolean;
+}
+
 /**
  * An AWS Lambda layer that includes the AWS CLI.
  */
 export class AwsCliLayer extends lambda.LayerVersion {
-  constructor(scope: Construct, id: string) {
-    super(scope, id, {
-      code: lambda.Code.fromAsset(path.join(__dirname, 'layer.zip'), {
+  /**
+   * @internal
+   */
+  public static _tryLoadPackage(targetVersion: string, log: (s: any) => void): any {
+    let availableVersion;
+    try {
+      const assetPackagePath = require.resolve(`${AwsCliLayer.assetPackageName}`);
+      availableVersion = AwsCliLayer.requireWrapper(path.join(assetPackagePath, '../../package.json'), log).version;
+    } catch (err) {
+      log('require.resolve error');
+    }
+
+    if (targetVersion === availableVersion) {
+      return AwsCliLayer.requireWrapper(AwsCliLayer.assetPackageName, log);
+    }
+  }
+
+  /**
+   * @internal
+   */
+  public static _downloadPackage(targetVersion: string, log: (s: string) => void): string | undefined {
+    const cdkHomeDir = cxapi.cdkHomeDir();
+    const downloadDir = path.join(cdkHomeDir, 'npm-cache');
+    const downloadPath = path.join(downloadDir, `${AwsCliLayer.assetPackageNpmTarPrefix}${targetVersion}.tgz`);
+    log(downloadPath);
+
+    if (fs.existsSync(downloadPath)) {
+      return downloadPath;
+    }
+    childproc.execSync(`mkdir -p ${downloadDir}; cd ${downloadDir}; npm pack ${AwsCliLayer.assetPackageName}@${targetVersion} -q`);
+    if (fs.existsSync(downloadPath)) {
+      return downloadPath;
+    }
+
+    return undefined;
+  }
+
+  private static readonly assetPackageName: string = '@aws-cdk/asset-awscli-v1';
+  private static readonly assetPackageNpmTarPrefix: string = 'aws-cdk-asset-awscli-v1-';
+
+  private static requireWrapper(id: string, log: (s: any) => void): any {
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      return require(id);
+    } catch (err) {
+      log(`require('${id}') failed`);
+      log(err);
+      if (err instanceof Error) {
+        console.error(err.name, err.message.split('\n')[0]);
+      }
+    }
+  }
+
+  private static installAndLoadPackage(from: string, log: (s: any) => void): any {
+    const installDir = AwsCliLayer.findInstallDir(log);
+    if (!installDir) {
+      return;
+    }
+    log(`install dir: ${installDir}`);
+    childproc.execSync(`npm install ${from} --no-save --prefix ${installDir} -q`);
+    return AwsCliLayer.requireWrapper(path.join(installDir, 'node_modules', AwsCliLayer.assetPackageName, 'lib/index.js'), log);
+  }
+
+  private static findInstallDir(log: (s: any) => void): string | undefined {
+    if (!require.main?.paths) {
+      return undefined;
+    }
+    for (let p of require.main.paths) {
+      log(`p: ${p}`);
+      if (fs.existsSync(p)) {
+        return p;
+      }
+    }
+    return undefined;
+  }
+
+  constructor(scope: Construct, id: string, props?: AwsCliLayerProps) {
+    const quiet = props?.quiet ?? true;
+    const log = (s: any) => {
+      if (!quiet) {
+        console.log(s);
+      }
+    };
+
+    const targetVersion = AwsCliLayer.requireWrapper(path.join(__dirname, '../package.json'), log).devDependencies[AwsCliLayer.assetPackageName];
+
+    let assetPackage;
+
+    let downloadStyle = 'require';
+    log('trying regular require');
+    assetPackage = AwsCliLayer._tryLoadPackage(targetVersion, log);
+
+    if (!assetPackage) {
+      downloadStyle = 'dynamic';
+      log('trying to download package');
+      const downloadPath = AwsCliLayer._downloadPackage(targetVersion, log);
+      if (downloadPath) {
+        log('trying to load from install location');
+        assetPackage = AwsCliLayer.installAndLoadPackage(downloadPath, log);
+      }
+    }
+
+    let code;
+    if (assetPackage) {
+      const asset = new assetPackage.AwsCliAsset(scope, `${id}-asset`);
+      code = lambda.Code.fromBucket(asset.bucket, asset.s3ObjectKey);
+    } else {
+      downloadStyle = 'fallback';
+      log('using fallback to original version');
+      code = lambda.Code.fromAsset(path.join(__dirname, 'layer.zip'), {
         // we hash the layer directory (it contains the tools versions and Dockerfile) because hashing the zip is non-deterministic
         assetHash: FileSystem.fingerprint(path.join(__dirname, '../layer')),
-      }),
+      });
+    }
+
+    super(scope, id, {
+      code: code,
       description: '/opt/awscli/aws',
     });
+
+    if (downloadStyle === 'fallback') {
+      log('we used the fallback when creating this construct, so a marker construct should be added to the tree for CLI notices');
+    }
   }
 }

packages/@aws-cdk/lambda-layer-awscli/package.json

@@ -78,18 +78,23 @@
     "@aws-cdk/integ-runner": "0.0.0",
     "@aws-cdk/integ-tests": "0.0.0",
     "@aws-cdk/pkglint": "0.0.0",
+    "@aws-cdk/asset-awscli-v1": "0.0.13",
     "@types/jest": "^27.5.2",
     "jest": "^27.5.1"
   },
   "dependencies": {
     "@aws-cdk/aws-lambda": "0.0.0",
+    "@aws-cdk/aws-s3-assets": "0.0.0",
     "@aws-cdk/core": "0.0.0",
+    "@aws-cdk/cx-api": "0.0.0",
     "constructs": "^10.0.0"
   },
   "homepage": "https://github.com/aws/aws-cdk",
   "peerDependencies": {
     "@aws-cdk/aws-lambda": "0.0.0",
+    "@aws-cdk/aws-s3-assets": "0.0.0",
     "@aws-cdk/core": "0.0.0",
+    "@aws-cdk/cx-api": "0.0.0",
     "constructs": "^10.0.0"
   },
   "engines": {
@@ -102,7 +107,7 @@
   },
   "cdk-build": {
     "pre": [
-      "layer/build.sh"
+      "layer/download-fallback.sh"
     ],
     "env": {
       "AWSLINT_BASE_CONSTRUCT": true
@@ -120,5 +125,10 @@
   "publishConfig": {
     "tag": "latest"
   },
+  "awslint": {
+    "exclude": [
+      "props-physical-name:@aws-cdk/lambda-layer-awscli.AwsCliLayerProps"
+    ]
+  },
   "private": true
 }