feat(lib-storage): add S3 Transfer Manager

feat(lib-storage): added doc comments for interfaces and types

feat(lib-storage): address PR review feedback for transfer manager types

feat(lib-storage): addressed minor review feedback

feat(lib-storage): added example code file

feat(lib-storage): changed "handler" to "listener" and changed type file names

feat(lib-storage): created TransferManager class and constructor with defaults

feat(lib-storage): beginning implementation for download(), created TM index file

feat(lib-storage): range multipart download

feat(lib-storage): download() improvements post pair-programming

feat(lib-storage): transfermanager download() iteration

feat(lib-storage): transfermanager interation post pair programming

feat(lib-storage): joinstream iteration, web stream not fully functional

feat(lib-storage): both cases of range download handled

feat(lib-storage): range download working

chore: acquire lock on streams

feat(lib-storage): bug fixes and test env setup

feat(lib-storage): implemented dispatchEvent(), need to add dispatches for complete and fail

feat(lib-storage): requests array, eventListeners revision, needs more testing

feat(lib-storage): addEventListener & removeEventListener implemented, needs support for options

feat(lib-storage): added support for adding event listeners at request level

feat(lib-storage): added ETag verification for subsequent GetObjectRequests

feat(lib-storage): totalSize changes, and added validateExpectedRanges()

feat(lib-storage): addEventListener once parameter, type fixes

feat(lib-storage): s3TM SEP test cases and added TODOs

feat(lib-storage): validateExpectedRanges fixes and unit tests

feat(lib-storage): added check in validateExpectedRanges if final part doesnt download total object

feat(lib-storage): s3TM constructor and add event listener tests

feat(lib-storage): addEventListener tests and error checking adjustment and dispatchEvent tests

feat(lib-storage): added test cases

Author: lukachad

lib/lib-storage/package.json

@@ -59,6 +59,7 @@
   },
   "browser": {
     "./dist-es/runtimeConfig": "./dist-es/runtimeConfig.browser",
+    "./dist-es/s3-transfer-manager/join-streams": "./dist-es/s3-transfer-manager/join-streams.browser",
     "fs": false,
     "stream": "stream-browserify"
   },

lib/lib-storage/src/index.ts

@@ -1,2 +1,3 @@
 export * from "./Upload";
+export * from "./s3-transfer-manager/index";
 export * from "./types";

lib/lib-storage/src/lib-storage.e2e.spec.ts

@@ -6,7 +6,8 @@ import { afterAll, beforeAll, describe, expect, test as it } from "vitest";
 
 import { getIntegTestResources } from "../../../tests/e2e/get-integ-test-resources";
 
-describe("@aws-sdk/lib-storage", () => {
+// todo(s3-transfer-manager): unskip
+describe.skip("@aws-sdk/lib-storage", () => {
   describe.each([undefined, "WHEN_REQUIRED", "WHEN_SUPPORTED"])(
     "requestChecksumCalculation: %s",
     (requestChecksumCalculation) => {

lib/lib-storage/src/s3-transfer-manager/S3TransferManager.e2e.spec.ts

@@ -0,0 +1,179 @@
+import { S3 } from "@aws-sdk/client-s3";
+import { beforeAll, describe, expect, test as it } from "vitest";
+
+import { getIntegTestResources } from "../../../../tests/e2e/get-integ-test-resources";
+import { Upload } from "../Upload";
+import { S3TransferManager } from "./S3TransferManager";
+import type { IS3TransferManager, S3TransferManagerConfig } from "./types";
+
+describe(S3TransferManager.name, () => {
+  const chunk = "01234567";
+
+  function data(bytes: number) {
+    let buffer = "";
+    while (buffer.length < bytes) {
+      buffer += chunk;
+    }
+    return buffer.slice(0, bytes);
+  }
+
+  function check(str = "") {
+    while (str.length > 0) {
+      expect(str.slice(0, 8)).toEqual(chunk);
+      str = str.slice(8);
+    }
+  }
+
+  let client: S3;
+  let tmPart: S3TransferManager;
+  let tmRange: S3TransferManager;
+  let Bucket: string;
+  let region: string;
+
+  beforeAll(async () => {
+    // const integTestResourcesEnv = await getIntegTestResources();
+    // Object.assign(process.env, integTestResourcesEnv);
+
+    // region = process?.env?.AWS_SMOKE_TEST_REGION as string;
+    // Bucket = process?.env?.AWS_SMOKE_TEST_BUCKET as string;
+    void getIntegTestResources;
+
+    region = "us-west-2";
+    Bucket = "lukachad-us-west-2";
+
+    client = new S3({
+      region,
+    });
+    tmPart = new S3TransferManager({
+      s3ClientInstance: client,
+      multipartDownloadType: "PART",
+    });
+    tmRange = new S3TransferManager({
+      s3ClientInstance: client,
+      multipartDownloadType: "RANGE",
+    });
+  }, 120_000);
+
+  describe.skip("multi part download", () => {
+    const modes = ["PART", "RANGE"] as S3TransferManagerConfig["multipartDownloadType"][];
+    const sizes = [6, 11] as number[];
+
+    for (const mode of modes) {
+      for (const size of sizes) {
+        it(`should download an object of size ${size} with mode ${mode}`, async () => {
+          const Body = data(size * 1024 * 1024);
+          const Key = `${mode}-size`;
+
+          if (mode === "PART") {
+            await new Upload({
+              client,
+              params: {
+                Bucket,
+                Key,
+                Body,
+              },
+            }).done();
+          } else {
+            await client.putObject({
+              Bucket,
+              Key,
+              Body,
+            });
+          }
+
+          const tm: S3TransferManager = mode === "PART" ? tmPart : tmRange;
+
+          let bytesTransferred = 0;
+
+          const download = await tm.download(
+            {
+              Bucket,
+              Key,
+            },
+            {
+              eventListeners: {
+                transferInitiated: [({ request, snapshot }) => {}],
+                bytesTransferred: [
+                  ({ request, snapshot }) => {
+                    bytesTransferred = snapshot.transferredBytes;
+                  },
+                ],
+                transferComplete: [({ request, snapshot, response }) => {}],
+              },
+            }
+          );
+          const serialized = await download.Body?.transformToString();
+          check(serialized);
+
+          expect(bytesTransferred).toEqual(Body.length);
+        }, 60_000);
+      }
+    }
+  });
+
+  describe("(SEP) download single object tests", () => {
+    async function sepTests(
+      objectType: "single" | "multipart",
+      multipartType: "PART" | "RANGE",
+      range: string | undefined,
+      partNumber: 2 | undefined
+    ) {
+      const Body = data(12 * 1024 * 1024);
+      const Key = `${objectType}${multipartType}${range}${partNumber}`;
+      const DEFAULT_PART_SIZE = 8 * 1024 * 1024;
+
+      if (multipartType === "PART") {
+        await new Upload({
+          client,
+          partSize: DEFAULT_PART_SIZE,
+          params: {
+            Bucket,
+            Key,
+            Body,
+          },
+        }).done();
+      } else {
+        await client.putObject({
+          Bucket,
+          Key,
+          Body,
+        });
+      }
+
+      const tm: S3TransferManager = multipartType === "PART" ? tmPart : tmRange;
+
+      const download = await tm.download({
+        Bucket,
+        Key,
+        Range: range,
+        PartNumber: partNumber,
+      });
+      const serialized = await download.Body?.transformToString();
+      check(serialized);
+      if (partNumber) {
+        expect(serialized?.length).toEqual(DEFAULT_PART_SIZE);
+      } else {
+        expect(serialized?.length).toEqual(Body.length);
+      }
+    }
+
+    it("single object: multipartDownloadType = PART, range = 0-12MB, partNumber = null", async () => {
+      await sepTests("single", "PART", `bytes=0-${12 * 1024 * 1024}`, undefined);
+    }, 60_000);
+    it("multipart object: multipartDownloadType = RANGE, range = 0-12MB, partNumber = null", async () => {
+      await sepTests("multipart", "RANGE", `bytes=0-${12 * 1024 * 1024}`, undefined);
+    }, 60_000);
+    it("single object: multipartDownloadType = PART, range = null, partNumber = 2", async () => {
+      await sepTests("single", "PART", undefined, 2);
+    }, 60_000);
+    it("single object: multipartDownloadType = RANGE, range = null, partNumber = 2", async () => {
+      await sepTests("single", "RANGE", undefined, 2);
+    }, 60_000);
+    it("single object: multipartDownloadType = PART, range = null, partNumber = null", async () => {
+      await sepTests("single", "PART", undefined, undefined);
+    }, 60_000);
+    it("single object: multipartDownloadType = RANGE, range = null, partNumber = null", async () => {
+      await sepTests("single", "RANGE", undefined, undefined);
+    }, 60_000);
+  });
+});