feat: Speed up and clean up code for adding and removing infrastructure

Author: ServerlessLife

.github/workflows/common-test.yml

@@ -23,7 +23,6 @@ permissions:
   contents: write
 
 env:
-  DISABLE_PARALLEL_DEPLOY: false
   REAL_NPM: ${{ inputs.mode == 'global' || inputs.mode == 'local' }}
   TEST_MONOREPO: ${{ inputs.testMonorepo }}
   node_version: 22
@@ -74,14 +73,14 @@ jobs:
         run: npm run deploy
         working-directory: test/cdk-basic
       - name: Test
-        run: npx vitest --retry 1 test/cdk-basic.test.ts
+        run: npx vitest --retry 2 test/cdk-basic.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/cdk-basic.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/cdk-basic.test.ts
       - name: Deploy YAML version
         run: npm run deploy-yaml
         working-directory: test/cdk-basic
       - name: Test YAML
-        run: npx vitest --retry 1 test/cdk-basic.test.ts
+        run: npx vitest --retry 2 test/cdk-basic.test.ts
 
   test-cdk-esm:
     runs-on: ubuntu-latest
@@ -128,9 +127,9 @@ jobs:
         run: npm run deploy
         working-directory: test/cdk-esm
       - name: Test
-        run: npx vitest --retry 1 test/cdk-esm.test.ts
+        run: npx vitest --retry 2 test/cdk-esm.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/cdk-esm.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/cdk-esm.test.ts
 
   test-sls-basic:
     runs-on: ubuntu-latest
@@ -178,9 +177,9 @@ jobs:
         run: npm run deploy
         working-directory: test/sls-basic
       - name: Test
-        run: npx vitest --retry 1 test/sls-basic.test.ts
+        run: npx vitest --retry 2 test/sls-basic.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/sls-basic.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/sls-basic.test.ts
 
   test-sls-esbuild-cjs:
     runs-on: ubuntu-latest
@@ -228,9 +227,9 @@ jobs:
         run: npm run deploy
         working-directory: test/sls-esbuild-cjs
       - name: Test
-        run: npx vitest --retry 1 test/sls-esbuild-cjs.test.ts
+        run: npx vitest --retry 2 test/sls-esbuild-cjs.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/sls-esbuild-cjs.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/sls-esbuild-cjs.test.ts
 
   test-sls-esbuild-esm:
     runs-on: ubuntu-latest
@@ -278,9 +277,9 @@ jobs:
         run: npm run deploy
         working-directory: test/sls-esbuild-esm
       - name: Test
-        run: npx vitest --retry 1 test/sls-esbuild-esm.test.ts
+        run: npx vitest --retry 2 test/sls-esbuild-esm.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/sls-esbuild-esm.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/sls-esbuild-esm.test.ts
 
   test-osls-basic:
     runs-on: ubuntu-latest
@@ -328,9 +327,9 @@ jobs:
         run: npm run deploy
         working-directory: test/osls-basic
       - name: Test
-        run: npx vitest --retry 1 test/osls-basic.test.ts
+        run: npx vitest --retry 2 test/osls-basic.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/osls-basic.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/osls-basic.test.ts
 
   test-osls-esbuild-cjs:
     runs-on: ubuntu-latest
@@ -378,9 +377,9 @@ jobs:
         run: npm run deploy
         working-directory: test/osls-esbuild-cjs
       - name: Test
-        run: npx vitest --retry 1 test/osls-esbuild-cjs.test.ts
+        run: npx vitest --retry 2 test/osls-esbuild-cjs.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/osls-esbuild-cjs.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/osls-esbuild-cjs.test.ts
 
   test-osls-esbuild-esm:
     runs-on: ubuntu-latest
@@ -428,9 +427,9 @@ jobs:
         run: npm run deploy
         working-directory: test/osls-esbuild-esm
       - name: Test
-        run: npx vitest --retry 1 test/osls-esbuild-esm.test.ts
+        run: npx vitest --retry 2 test/osls-esbuild-esm.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/osls-esbuild-esm.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/osls-esbuild-esm.test.ts
 
   test-sam-basic:
     runs-on: ubuntu-latest
@@ -481,9 +480,9 @@ jobs:
         run: npm run deploy
         working-directory: test/sam-basic
       - name: Test
-        run: npx vitest --retry 1 test/sam-basic.test.ts
+        run: npx vitest --retry 2 test/sam-basic.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/sam-basic.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/sam-basic.test.ts
 
   test-sam-alt:
     runs-on: ubuntu-latest
@@ -534,9 +533,9 @@ jobs:
         run: npm run deploy
         working-directory: test/sam-alt
       - name: Test
-        run: npx vitest --retry 1 test/sam-alt.test.ts
+        run: npx vitest --retry 2 test/sam-alt.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/sam-alt.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/sam-alt.test.ts
 
   test-terraform-basic:
     runs-on: ubuntu-latest
@@ -590,9 +589,9 @@ jobs:
         run: npm run deploy
         working-directory: test/terraform-basic
       - name: Test
-        run: npx vitest --retry 1 test/terraform-basic.test.ts
+        run: npx vitest --retry 2 test/terraform-basic.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/terraform-basic.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/terraform-basic.test.ts
 
   test-opentofu-basic:
     runs-on: ubuntu-latest
@@ -646,6 +645,6 @@ jobs:
         run: npm run deploy
         working-directory: test/opentofu-basic
       - name: Test
-        run: npx vitest --retry 1 test/opentofu-basic.test.ts
+        run: npx vitest --retry 2 test/opentofu-basic.test.ts
       - name: Test - observability mode
-        run: OBSERVABLE_MODE=true npx vitest --retry 1 test/opentofu-basic.test.ts
+        run: OBSERVABLE_MODE=true npx vitest --retry 2 test/opentofu-basic.test.ts

.github/workflows/publish.yml

@@ -6,16 +6,6 @@ run-name: >
 
 on:
   workflow_dispatch:
-    inputs:
-      release-type:
-        description: 'Release type'
-        required: true
-        default: 'stable'
-        type: choice
-        options:
-          - stable
-          - alpha
-          - beta
 
 permissions:
   id-token: write

.github/workflows/pull-request.yml

@@ -15,10 +15,41 @@ jobs:
     uses: ./.github/workflows/common-build.yml
     secrets: inherit
 
+  remove-old-layers:
+    runs-on: ubuntu-latest
+    needs: build
+    concurrency:
+      group: remove-all-layers
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.node_version }}
+          registry-url: 'https://registry.npmjs.org'
+      - name: Install dependencies
+        run: npm ci
+      - name: Download build artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-west-1
+          role-to-assume: ${{ secrets.AWS_ROLE }}
+          role-session-name: GitHubActions
+      - name: Remove old layers
+        run: |
+          node ../../dist/lldebugger.mjs -r all --config-env=test -v
+        # Picking random test so I can have environment
+        working-directory: test/sam-basic
+
   test:
     uses: ./.github/workflows/common-test.yml
     secrets: inherit
-    needs: build
+    needs: remove-old-layers
     with:
       mode: build
       testMonorepo: false

README.md

@@ -21,7 +21,7 @@ Serverless is amazing and solves many issues with traditional systems. However,
 
 Lambda Live Debugger connects to your deployed Lambda, routes requests to your computer, and sends responses back to the deployed Lambda. This allows you to debug locally, but the system behaves as if the code is running in the cloud with the same permissions. In case of code changes, you do not have to redeploy. The code is reloaded automatically without deploying or even restarting the debugger.
 
-The tool attaches Lambda Extensions (via a Layer), intercepts, and relays calls to AWS IoT. AWS IoT transfers messages between your Lambda and the local machine. If the Lambda is written in TypeScript, it's transpiled to JavaScript. The code is executed via the Node Worker Thread.
+The tool attaches Lambda Extensions (via a layer), intercepts, and relays calls to AWS IoT. AWS IoT transfers messages between your Lambda and the local machine. If the Lambda is written in TypeScript, it's transpiled to JavaScript. The code is executed via the Node Worker Thread.
 
 ![Architecture](./public/architecture.drawio.png)
 
@@ -34,11 +34,11 @@ AWS keys generated on the cloud for Lambda are transferred to the local environm
 
 Lambda Live Debugger makes the following changes to your AWS infrastructure:
 
-- Deploys the Lambda Layer
-- Attaches the Layer to each Lambda you're debugging
+- Deploys the Lambda layer
+- Attaches the layer to each Lambda you're debugging
 - Adds a policy to the Lambda Role for AWS IoT access
 
-In case you do not want to debug all functions and add the Layer to them, you can limit to the ones you need via the `function` parameter.
+In case you do not want to debug all functions and add the layer to them, you can limit to the ones you need via the `function` parameter.
 
 The tool generates temporary files in the `.lldebugger` folder, which can be deleted after debugging. The wizard can add `.lldebugger` to `.gitignore` for you.
 
@@ -109,7 +109,7 @@ The configuration is saved to `lldebugger.config.ts`.
 
 ```
  -V, --version                   output the version number
- -r, --remove [option]           Remove Lambda Live Debugger infrastructure. Options: 'keep-layer' (default), 'all'. The latest also removes the Lambda Layer
+ -r, --remove [option]           Remove Lambda Live Debugger infrastructure. Options: 'keep-layer' (default), 'all'. The latest also removes the Lambda layer
  -w, --wizard                    Program interactively asks for each parameter and saves it to lldebugger.config.ts
  -v, --verbose                   Verbose logging
  -c, --context <context>         AWS CDK context (default: [])
@@ -187,9 +187,9 @@ When you no longer want to debug and want Lambda to execute the code deployed to
 lld -r
 ```
 
-This detaches the Layer from your Lambdas and removes the IoT permission policy. It will not remove the Layer because others might use it.
+This detaches the layer from your Lambdas and removes the IoT permission policy. It will not remove the layer because others might use it.
 
-To also remove the Layer:
+To also remove the layer:
 
 ```
 lld -r=all

src/configuration.ts

@@ -1,4 +1,3 @@
-import { LambdaProps } from './types/lambdaProps.js';
 import { LldConfig } from './types/lldConfig.js';
 import { LambdaResource } from './types/resourcesDiscovery.js';
 import * as crypto from 'crypto';
@@ -12,7 +11,7 @@ import { ResourceDiscovery } from './resourceDiscovery.js';
 import { Logger } from './logger.js';
 
 let config: LldConfig;
-const lambdas: Record<string, LambdaProps> = {};
+const lambdas: Record<string, LambdaResource> = {};
 let lambdasList: LambdaResource[] | undefined = undefined;
 
 /**
@@ -103,34 +102,43 @@ async function generateDebuggerId(observableMode: boolean) {
  * Add a Lambda to the configuration
  * @param props
  */
-function addLambda(props: Omit<LambdaProps, 'functionId'>) {
+function addLambda(props: LambdaResource) {
   lambdas[props.functionName] = {
-    functionId: props.functionName,
     ...props,
   };
 }
 
 /**
- * Get a Lambda by functionId
- * @param functionId
+ * Get a Lambda by name
+ * @param functionName
  * @returns
  */
-async function getLambda(functionId: string): Promise<LambdaProps> {
-  const lambda = lambdas[functionId];
+async function getLambda(functionName: string): Promise<LambdaResource> {
+  const lambda = lambdas[functionName];
 
   if (lambda) return lambda;
 
-  throw new Error(`Lambda not found: ${functionId}`);
+  throw new Error(`Lambda not found: ${functionName}`);
 }
 
 /**
  * Get all Lambdas
  * @returns
  */
-function getLambdas() {
+function getLambdasAll(): LambdaResource[] {
   return Object.values(lambdas);
 }
 
+/**
+ * Get filtered Lambdas
+ * @returns
+ */
+function getLambdasFiltered() {
+  const list = Object.values(lambdas);
+
+  return list.filter((l) => !l.filteredOut);
+}
+
 /**
  * Discover Lambdas
  */
@@ -180,7 +188,7 @@ function saveDiscoveredLambdas(lambdasListNew: LambdaResource[]) {
 
     Logger.log('Found the following Lambdas to debug:');
     Logger.log(
-      ` - ${getLambdas()
+      ` - ${getLambdasFiltered()
         .map((f) => `${f.functionName} code: ${f.codePath}`)
         .join('\n - ')}`,
     );
@@ -205,6 +213,7 @@ export const Configuration = {
   },
   discoverLambdas,
   getLambda,
-  getLambdas,
+  getLambdasAll,
+  getLambdasFiltered,
   setConfig,
 };

src/configuration/getConfigFromCliArgs.ts

@@ -20,7 +20,7 @@ export async function getConfigFromCliArgs(
   program.name('lld').description('Lambda Live Debugger').version(version);
   program.option(
     '-r, --remove [option]',
-    "Remove Lambda Live Debugger infrastructure. Options: 'keep-layer' (default), 'all'. The latest also removes the Lambda Layer",
+    "Remove Lambda Live Debugger infrastructure. Options: 'keep-layer' (default), 'all'. The latest also removes the Lambda layer",
     //validateRemoveOption,
     //"keep-layer"
   );