fix: stager fails when the file to download doesn't exist

.projenrc.ts

@@ -65,6 +65,7 @@ const project = new CdklabsConstructLibrary({
     'tar-stream',
     'uuid',
     'yaml',
+    'nock',
     'normalize-registry-metadata',
     '@octokit/rest',
     'markdown-it',

src/__tests__/package-sources/npmjs/stage-and-notify.lambda.test.ts

@@ -0,0 +1,59 @@
+import { Context } from 'aws-lambda';
+import * as AWS from 'aws-sdk';
+import * as AWSMock from 'aws-sdk-mock';
+import * as nock from 'nock';
+import {
+  ENV_DENY_LIST_BUCKET_NAME,
+  ENV_DENY_LIST_OBJECT_KEY,
+} from '../../../backend/deny-list/constants';
+import {
+  handler,
+  PackageVersion,
+} from '../../../package-sources/npmjs/stage-and-notify.lambda';
+
+const MOCK_DENY_LIST_BUCKET = 'deny-list-bucket-name';
+const MOCK_DENY_LIST_OBJECT = 'my-deny-list.json';
+
+type Response<T> = (err: AWS.AWSError | null, data?: T) => void;
+
+beforeEach(() => {
+  process.env.BUCKET_NAME = 'foo';
+  process.env.QUEUE_URL = 'bar';
+  process.env[ENV_DENY_LIST_BUCKET_NAME] = MOCK_DENY_LIST_BUCKET;
+  process.env[ENV_DENY_LIST_OBJECT_KEY] = MOCK_DENY_LIST_OBJECT;
+});
+
+afterEach(() => {
+  process.env.BUCKET_NAME = undefined;
+  process.env.QUEUE_URL = undefined;
+  delete process.env[ENV_DENY_LIST_BUCKET_NAME];
+  delete process.env[ENV_DENY_LIST_OBJECT_KEY];
+});
+
+test('ignores 404', async () => {
+  const basePath = 'https://registry.npmjs.org';
+  const uri = '/@pepperize/cdk-vpc/-/cdk-vpc-0.0.785.tgz';
+
+  AWSMock.mock(
+    'S3',
+    'getObject',
+    (_req: AWS.S3.GetObjectRequest, cb: Response<AWS.S3.GetObjectOutput>) => {
+      cb(null, { Body: JSON.stringify({}) });
+    }
+  );
+
+  nock(basePath).get(uri).reply(404);
+
+  const event: PackageVersion = {
+    tarballUrl: `${basePath}${uri}`,
+    integrity: '09d37ec93c5518bf4842ac8e381a5c06452500e5',
+    modified: '2023-09-22T15:48:10.381Z',
+    name: '@pepper/cdk-vpc',
+    seq: '26437963',
+    version: '0.0.785',
+  };
+
+  const context: Context = {} as any;
+
+  await expect(handler(event, context)).resolves.toBe(undefined);
+});

src/package-sources/npmjs/stage-and-notify.lambda.ts

@@ -7,6 +7,8 @@ import { s3, sqs } from '../../backend/shared/aws.lambda-shared';
 import { requireEnv } from '../../backend/shared/env.lambda-shared';
 import { integrity } from '../../backend/shared/integrity.lambda-shared';
 
+class HttpNotFoundError extends Error {}
+
 /**
  * This function is invoked by the `npm-js-follower.lambda`  with a `PackageVersion` object, or by
  * an SQS trigger feeding from this function's Dead-Letter Queue (for re-trying purposes).
@@ -38,9 +40,23 @@ export async function handler(
     return;
   }
 
-  // Download the tarball
-  console.log(`Downloading tarball from URL: ${event.tarballUrl}`);
-  const tarball = await httpGet(event.tarballUrl);
+  let tarball: Buffer = Buffer.from('');
+  try {
+    // Download the tarball
+    console.log(`Downloading tarball from URL: ${event.tarballUrl}`);
+    tarball = await httpGet(event.tarballUrl);
+  } catch (e) {
+    if (e instanceof HttpNotFoundError) {
+      // We received a message to download a file that should exist but doesn't.
+      // If we throw an error, the message will be sent to the DLQ, to be processed
+      // again in a re-drive. But given that this file will probably never be
+      // available, the re-drives will also fail, and we'll never be able to get rid
+      // of the message in the DLQ. Instead, we ignore this version by returning silently.
+      return;
+    } else {
+      throw e;
+    }
+  }
 
   // Store the tarball into the staging bucket
   // - infos.dist.tarball => https://registry.npmjs.org/<@scope>/<name>/-/<name>-<version>.tgz
@@ -135,7 +151,10 @@ export interface PackageVersion {
 function httpGet(url: string) {
   return new Promise<Buffer>((ok, ko) => {
     https.get(url, (response) => {
-      if (response.statusCode !== 200) {
+      console.log(response.statusCode);
+      if (response.statusCode === 404) {
+        ko(new HttpNotFoundError());
+      } else if (response.statusCode !== 200) {
         ko(
           new Error(
             `Unsuccessful GET: ${response.statusCode} - ${response.statusMessage}`