feat: optional checksum algorithm for upload (aws-amplify#13849)

* feat: opt in checksum * fix: revert local prettier suggestion * fix: up size limit for storage upload data * feat: react native crc32 * fix: up bundle size limit and fix typo * feat: add documentation for checksumAlgorithm * fix: update bundle size limit * fix: update bundle size limit * fix: address pr feedbacks * fix: bundle-size limit --------- Co-authored-by: AllanZhengYP <zheallan@amazon.com>
AllanZhengYP · Oct 9, 2024 · 02cb08a · 02cb08a
1 parent 7318ba2
commit 02cb08a
Show file tree

Hide file tree

Showing 17 changed files with 697 additions and 125 deletions.
diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json
@@ -497,7 +497,7 @@
  "name": "[Storage] uploadData (S3)",
  "path": "./dist/esm/storage/index.mjs",
  "import": "{ uploadData }",
- "limit": "22.07 kB"
+ "limit": "22.16 kB"
  }
  ]
 }
diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts
@@ -20,7 +20,10 @@ import {
  StorageValidationErrorCode,
  validationErrorMap,
 } from '../../../../../src/errors/types/validation';
-import { UPLOADS_STORAGE_KEY } from '../../../../../src/providers/s3/utils/constants';
+import {
+ CHECKSUM_ALGORITHM_CRC32,
+ UPLOADS_STORAGE_KEY,
+} from '../../../../../src/providers/s3/utils/constants';
 import { byteLength } from '../../../../../src/providers/s3/apis/uploadData/byteLength';
 import { CanceledError } from '../../../../../src/errors/CanceledError';
 import { StorageOptions } from '../../../../../src/types';
@@ -47,9 +50,15 @@ const bucket = 'bucket';
 const region = 'region';
 const defaultKey = 'key';
 const defaultContentType = 'application/octet-stream';
-const defaultCacheKey = '8388608_application/octet-stream_bucket_public_key';
+const defaultCacheKey =
+ '/twwTw==_8388608_application/octet-stream_bucket_public_key';
 const testPath = 'testPath/object';
-const testPathCacheKey = `8388608_${defaultContentType}_${bucket}_custom_${testPath}`;
+const testPathCacheKey = `/twwTw==_8388608_${defaultContentType}_${bucket}_custom_${testPath}`;
+
+const generateTestPathCacheKey = (optionsHash: string) =>
+ `${optionsHash}_8388608_${defaultContentType}_${bucket}_custom_${testPath}`;
+const generateDefaultCacheKey = (optionsHash: string) =>
+ `${optionsHash}_8388608_application/octet-stream_bucket_public_key`;
 
 const mockCreateMultipartUpload = jest.mocked(createMultipartUpload);
 const mockUploadPart = jest.mocked(uploadPart);
@@ -83,10 +92,6 @@ const mockCalculateContentCRC32Mock = () => {
  seed: 0,
  });
 };
-const mockCalculateContentCRC32Undefined = () => {
- mockCalculateContentCRC32.mockReset();
- mockCalculateContentCRC32.mockResolvedValue(undefined);
-};
 const mockCalculateContentCRC32Reset = () => {
  mockCalculateContentCRC32.mockReset();
  mockCalculateContentCRC32.mockImplementation(
@@ -291,6 +296,9 @@ describe('getMultipartUploadHandlers with key', () => {
  const { multipartUploadJob } = getMultipartUploadHandlers({
  key: defaultKey,
  data: twoPartsPayload,
+ options: {
+ checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32,
+ },
  });
  await multipartUploadJob();
 
@@ -301,9 +309,11 @@ describe('getMultipartUploadHandlers with key', () => {
  *
  * uploading each part calls calculateContentCRC32 1 time each
  *
- * these steps results in 5 calls in total
+ * 1 time for optionsHash
+ *
+ * these steps results in 6 calls in total
  */
- expect(calculateContentCRC32).toHaveBeenCalledTimes(5);
+ expect(calculateContentCRC32).toHaveBeenCalledTimes(6);
  expect(calculateContentMd5).not.toHaveBeenCalled();
  expect(mockUploadPart).toHaveBeenCalledTimes(2);
  expect(mockUploadPart).toHaveBeenCalledWith(
@@ -317,8 +327,7 @@ describe('getMultipartUploadHandlers with key', () => {
  },
  );
 
- it('should use md5 if crc32 is returning undefined', async () => {
- mockCalculateContentCRC32Undefined();
+ it('should use md5 if no using crc32', async () => {
  mockMultipartUploadSuccess();
  Amplify.libraryOptions = {
  Storage: {
@@ -372,6 +381,9 @@ describe('getMultipartUploadHandlers with key', () => {
  {
  key: defaultKey,
  data: file,
+ options: {
+ checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32,
+ },
  },
  file.size,
  );
@@ -589,7 +601,7 @@ describe('getMultipartUploadHandlers with key', () => {
  expect(Object.keys(cacheValue)).toEqual([
  expect.stringMatching(
  // \d{13} is the file lastModified property of a file
- /someName_\d{13}_8388608_application\/octet-stream_bucket_public_key/,
+ /someName_\d{13}_\/twwTw==_8388608_application\/octet-stream_bucket_public_key/,
  ),
  ]);
  });
@@ -800,7 +812,7 @@ describe('getMultipartUploadHandlers with key', () => {
  >;
  mockDefaultStorage.getItem.mockResolvedValue(
  JSON.stringify({
- [defaultCacheKey]: {
+ [generateDefaultCacheKey('o6a/Qw==')]: {
  uploadId: 'uploadId',
  bucket,
  key: defaultKey,
@@ -942,6 +954,9 @@ describe('getMultipartUploadHandlers with path', () => {
  const { multipartUploadJob } = getMultipartUploadHandlers({
  path: testPath,
  data: twoPartsPayload,
+ options: {
+ checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32,
+ },
  });
  await multipartUploadJob();
 
@@ -952,9 +967,11 @@ describe('getMultipartUploadHandlers with path', () => {
  *
  * uploading each part calls calculateContentCRC32 1 time each
  *
- * these steps results in 5 calls in total
+ * 1 time for optionsHash
+ *
+ * these steps results in 6 calls in total
  */
- expect(calculateContentCRC32).toHaveBeenCalledTimes(5);
+ expect(calculateContentCRC32).toHaveBeenCalledTimes(6);
  expect(calculateContentMd5).not.toHaveBeenCalled();
  expect(mockUploadPart).toHaveBeenCalledTimes(2);
  expect(mockUploadPart).toHaveBeenCalledWith(
@@ -968,8 +985,7 @@ describe('getMultipartUploadHandlers with path', () => {
  },
  );
 
- it('should use md5 if crc32 is returning undefined', async () => {
- mockCalculateContentCRC32Undefined();
+ it('should use md5 if no using crc32', async () => {
  mockMultipartUploadSuccess();
  Amplify.libraryOptions = {
  Storage: {
@@ -1023,6 +1039,9 @@ describe('getMultipartUploadHandlers with path', () => {
  {
  path: testPath,
  data: file,
+ options: {
+ checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32,
+ },
  },
  file.size,
  );
@@ -1533,9 +1552,10 @@ describe('getMultipartUploadHandlers with path', () => {
  const mockDefaultStorage = defaultStorage as jest.Mocked<
  typeof defaultStorage
  >;
+
  mockDefaultStorage.getItem.mockResolvedValue(
  JSON.stringify({
- [testPathCacheKey]: {
+ [generateTestPathCacheKey('o6a/Qw==')]: {
  uploadId: 'uploadId',
  bucket,
  key: testPath,