feat: Reduce aws-cdk-lib package size by not bundling awscli Lambda Layer

notes.txt

@@ -0,0 +1,11 @@
+Some notes about testing
+
+To verify the current code, I have created a standalone cdk application that has
+dependencies directly on locally created tarballs for aws-cdk-lib and
+@aws-cdk/lambda-layer-awscli which are generated from running `yarn package` in
+the corresponding directory.
+
+npm install doesn't like it when I change the contents of the tarball, and
+reinstall with the same file name. So I've manually incremented the version
+number at the end. This is not a good workflow. We need a better one for
+developing this project. 

packages/@aws-cdk/lambda-layer-awscli/lib/awscli-layer.ts

@@ -1,19 +1,82 @@
-import * as path from 'path';
+/* eslint-disable no-console */
 import * as lambda from '@aws-cdk/aws-lambda';
-import { FileSystem } from '@aws-cdk/core';
-import { Construct } from 'constructs';
+import { RemovalPolicy, ResourceEnvironment, Stack } from '@aws-cdk/core';
+import { Construct, Node } from 'constructs';
+import * as childproc from 'child_process';
+import * as os from 'os';
+import * as path from 'path';
 
 /**
  * An AWS Lambda layer that includes the AWS CLI.
  */
-export class AwsCliLayer extends lambda.LayerVersion {
+export class AwsCliLayer implements lambda.ILayerVersion {
+
+  public readonly env: ResourceEnvironment;
+  public readonly layerVersionArn: string;
+  public readonly node: Node;
+  public readonly stack: Stack;
+  public readonly compatibleRuntimes?: lambda.Runtime[];
+
+  private readonly packageName: string = 'lambda-layer-awscli-v1';
+  private readonly npmPackage: any;
+  private readonly layer: lambda.LayerVersion;
+  private readonly wellKnownInstallDir: string;
+
   constructor(scope: Construct, id: string) {
-    super(scope, id, {
-      code: lambda.Code.fromAsset(path.join(__dirname, 'layer.zip'), {
-        // we hash the layer directory (it contains the tools versions and Dockerfile) because hashing the zip is non-deterministic
-        assetHash: FileSystem.fingerprint(path.join(__dirname, '../layer')),
-      }),
-      description: '/opt/awscli/aws',
-    });
+    const installParentDir = os.homedir() ?? os.tmpdir();
+    this.wellKnownInstallDir = path.join(installParentDir, '.awscdk/npm-cache');
+
+    const version = this.requireWrapper(path.join(__dirname, '../package.json')).devDependencies[this.packageName];
+    const pathOfModuleIfAlreadyInstalled = require.resolve(`${this.packageName}`);
+    const versionAlreadyInstalled = this.requireWrapper(path.join(pathOfModuleIfAlreadyInstalled, '../../package.json')).version;
+
+    if (version !== versionAlreadyInstalled) {
+      this.npmPackage = this.installNpmPackage();
+    }
+    if (!this.npmPackage) {
+      this.npmPackage = this.requireWrapper(this.packageName);
+    }
+    if (!this.npmPackage) {
+      this.npmPackage = this.installNpmPackage();
+    }
+    if (!this.npmPackage) {
+      throw new Error(`Unable to load ${this.packageName}@${version}. See XYZ for details and how to work around this issue.`);
+    }
+
+    this.layer = new this.npmPackage.AwsCliLayer(scope, id);
+
+    this.env = this.layer.env;
+    this.layerVersionArn = this.layer.layerVersionArn;
+    this.node = this.layer.node;
+    this.stack = this.layer.stack;
+    this.compatibleRuntimes = this.layer.compatibleRuntimes;
+  }
+
+  public addPermission(id: string, permission: lambda.LayerVersionPermission): void {
+    this.layer.addPermission(id, permission);
+  }
+
+  public applyRemovalPolicy(policy: RemovalPolicy): void {
+    this.layer.applyRemovalPolicy(policy);
+  }
+
+  private requireWrapper(id: string): any {
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      return require(id);
+    } catch (err) {
+      console.log(`require('${id}') failed`);
+      console.log(err);
+      if (err instanceof Error) {
+        console.error(err.name, err.message.split('\n')[0]);
+      }
+    }
+  }
+
+  private installNpmPackage(): any {
+    console.log(`Shelling out to run npm install ${this.packageName} --no-save --prefix ${this.wellKnownInstallDir}`);
+    const result = childproc.execSync(`pwd; npm prefix; npm install  ${this.packageName} --silent --no-save --prefix ${this.wellKnownInstallDir}`);
+    console.log(result.toString('utf8'));
+    return this.requireWrapper(path.join(this.wellKnownInstallDir, 'node_modules', this.packageName, 'lib/index.js'));
   }
 }

packages/@aws-cdk/lambda-layer-awscli/package.json

@@ -72,19 +72,16 @@
   },
   "license": "Apache-2.0",
   "devDependencies": {
-    "@aws-cdk/assertions": "0.0.0",
-    "@aws-cdk/custom-resources": "0.0.0",
     "@aws-cdk/cdk-build-tools": "0.0.0",
     "@aws-cdk/integ-runner": "0.0.0",
     "@aws-cdk/integ-tests": "0.0.0",
     "@aws-cdk/pkglint": "0.0.0",
+    "constructs": "^10.0.0",
+    "lambda-layer-awscli-v1": "0.0.2",
     "@types/jest": "^27.5.2",
     "jest": "^27.5.1"
   },
   "dependencies": {
-    "@aws-cdk/aws-lambda": "0.0.0",
-    "@aws-cdk/core": "0.0.0",
-    "constructs": "^10.0.0"
   },
   "homepage": "https://github.com/aws/aws-cdk",
   "peerDependencies": {
@@ -101,16 +98,13 @@
     "announce": false
   },
   "cdk-build": {
-    "pre": [
-      "layer/build.sh"
-    ],
+    "eslint": {
+      "disable": true
+    },
     "env": {
       "AWSLINT_BASE_CONSTRUCT": true
     }
   },
-  "ubergen": {
-    "exclude": false
-  },
   "nozem": {
     "ostools": [
       "dirname",
@@ -119,6 +113,5 @@
   },
   "publishConfig": {
     "tag": "latest"
-  },
-  "private": true
+  }
 }

packages/@aws-cdk/lambda-layer-awscli/test/awscli-layer.test.ts

@@ -14,3 +14,17 @@ test('synthesized to a layer version', () => {
     Description: '/opt/awscli/aws',
   });
 });
+
+test('installing from npm', () => {
+  jest.mock('lambda-layer-awscli-v1', () => undefined);
+  //GIVEN
+  const stack = new Stack();
+
+  // WHEN
+  new AwsCliLayer(stack, 'MyLayer');
+
+  // THEN
+  Template.fromStack(stack).hasResourceProperties('AWS::Lambda::LayerVersion', {
+    Description: '/opt/awscli/aws',
+  });
+});

packages/aws-cdk-lib/.gitignore

@@ -12,4 +12,8 @@
 junit.xml
 !.eslintrc.js
 dist
-.LAST_PACKAGE
+.LAST_PACKAGE
+
+.nyc_output
+coverage
+nyc.config.js

packages/aws-cdk-lib/lambda-layer-awscli/lib/awscli-layer.ts

@@ -0,0 +1,43 @@
+/* eslint-disable no-console */
+import { Construct, Node } from 'constructs';
+import * as lambda from '../../aws-lambda';
+import { RemovalPolicy, ResourceEnvironment, Stack } from '../../core';
+
+/**
+ * An AWS Lambda layer that includes the AWS CLI.
+ */
+export class AwsCliLayer implements lambda.ILayerVersion {
+
+  private readonly package: any;
+  private readonly layer: lambda.LayerVersion;
+  public readonly layerVersionArn: string;
+  public readonly compatibleRuntimes?: lambda.Runtime[];
+  public readonly stack: Stack;
+  public readonly env: ResourceEnvironment;
+  public readonly node: Node;
+
+  constructor(scope: Construct, id: string) {
+    // eslint-disable-next-line no-console
+    console.log('loading package');
+    // automatically get this version from version.v2.json.version-1
+    console.log(require.resolve('@aws-cdk/lambda-layer-awscli@2.36.0'));
+    // eslint-disable-next-line @typescript-eslint/no-require-imports, import/no-extraneous-dependencies
+    this.package = require('@aws-cdk/lambda-layer-awscli');
+    console.log(Object.keys(this.package));
+
+    this.layer = new this.package.AwsCliLayer(scope, id);
+    this.layerVersionArn = this.layer.layerVersionArn;
+    this.compatibleRuntimes = this.layer.compatibleRuntimes;
+    this.stack = this.layer.stack;
+    this.env = this.layer.env;
+    this.node = this.layer.node;
+  }
+
+  public addPermission(id: string, permission: lambda.LayerVersionPermission): void {
+    this.layer.addPermission(id, permission);
+  }
+
+  applyRemovalPolicy(policy: RemovalPolicy): void {
+    this.layer.applyRemovalPolicy(policy);
+  }
+}

packages/aws-cdk-lib/package.json

@@ -15,7 +15,7 @@
     "gen": "ubergen",
     "build": "cdk-build",
     "lint": "cdk-lint",
-    "test": "echo done",
+    "test": "cdk-test",
     "package": "cdk-package",
     "pkglint": "pkglint -f",
     "build+test": "yarn build && yarn test",
@@ -360,10 +360,12 @@
     "@aws-cdk/ubergen": "0.0.0",
     "@types/fs-extra": "^8.1.2",
     "@types/node": "^14.18.26",
+    "@types/jest": "^27.5.2",
     "constructs": "^10.0.0",
     "esbuild": "^0.15.5",
     "fs-extra": "^9.1.0",
     "ts-node": "^9.1.1",
+    "jest": "^27.5.1",
     "typescript": "~3.8.3"
   },
   "peerDependencies": {

scripts/verify-aws-cdk-lib-with-separated-lambda-layers.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Pre-requisites:
+# - aws-cdk-lib is built and has the "wrapper" code
+# - @aws-cdk/lambda-layer-awscli is built
+
+STARTING_DIR=$(pwd)
+echo $STARTING_DIR
+
+REPO_ROOT=$(cd $(dirname $0) && pwd)/../
+
+echo $REPO_ROOT
+
+# authenticate with code artifact repo
+aws codeartifact login --tool npm --repository lambda-layer-separate-packages --domain mkusters
+
+cd $REPO_ROOT/packages/@aws-cdk/lambda-layer-awscli
+# yarn build
+# yarn package
+# upload to code artifacte repo
+# npm publish ./dist/js/lambda-layer-awscli\@0.0.0.jsii.tgz
+
+# build aws-cdk-lib with the "wrapper"
+cd $REPO_ROOT/packages/aws-cdk-lib/
+yarn build
+
+

tools/@aws-cdk/pkglint/lib/rules.ts

@@ -62,6 +62,7 @@ export class PublishConfigTagIsRequired extends ValidationRule {
     '@aws-cdk/cloudformation-diff',
     '@aws-cdk/cx-api',
     '@aws-cdk/region-info',
+    '@aws-cdk/lambda-layer-awscli',
     'aws-cdk',
     'awslint',
     'cdk-assets',
@@ -1665,6 +1666,7 @@ export class UbergenPackageVisibility extends ValidationRule {
     '@aws-cdk/cloudformation-diff',
     '@aws-cdk/cx-api',
     '@aws-cdk/region-info',
+    '@aws-cdk/lambda-layer-awscli',
     'aws-cdk-lib',
     'aws-cdk',
     'awslint',

tools/@aws-cdk/ubergen/bin/ubergen.ts

@@ -602,6 +602,7 @@ async function rewriteRosettaFixtureImports(fromFile: string, libName: string):
 }
 
 const IGNORED_FILE_NAMES = new Set([
+  'awscli-layer.ts',
   '.eslintrc.js',
   '.gitignore',
   '.jest.config.js',