diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 1cede57f7ad..8d2c0dbab3c 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "22.07 kB" + "limit": "22.16 kB" } ] } diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts index 695822f57c8..0796df5540e 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts @@ -20,7 +20,10 @@ import { StorageValidationErrorCode, validationErrorMap, } from '../../../../../src/errors/types/validation'; -import { UPLOADS_STORAGE_KEY } from '../../../../../src/providers/s3/utils/constants'; +import { + CHECKSUM_ALGORITHM_CRC32, + UPLOADS_STORAGE_KEY, +} from '../../../../../src/providers/s3/utils/constants'; import { byteLength } from '../../../../../src/providers/s3/apis/uploadData/byteLength'; import { CanceledError } from '../../../../../src/errors/CanceledError'; import { StorageOptions } from '../../../../../src/types'; @@ -47,9 +50,15 @@ const bucket = 'bucket'; const region = 'region'; const defaultKey = 'key'; const defaultContentType = 'application/octet-stream'; -const defaultCacheKey = '8388608_application/octet-stream_bucket_public_key'; +const defaultCacheKey = + '/twwTw==_8388608_application/octet-stream_bucket_public_key'; const testPath = 'testPath/object'; -const testPathCacheKey = `8388608_${defaultContentType}_${bucket}_custom_${testPath}`; +const testPathCacheKey = `/twwTw==_8388608_${defaultContentType}_${bucket}_custom_${testPath}`; + +const generateTestPathCacheKey = (optionsHash: string) => + `${optionsHash}_8388608_${defaultContentType}_${bucket}_custom_${testPath}`; +const generateDefaultCacheKey = (optionsHash: string) => + `${optionsHash}_8388608_application/octet-stream_bucket_public_key`; const mockCreateMultipartUpload = jest.mocked(createMultipartUpload); const mockUploadPart = jest.mocked(uploadPart); @@ -83,10 +92,6 @@ const mockCalculateContentCRC32Mock = () => { seed: 0, }); }; -const mockCalculateContentCRC32Undefined = () => { - mockCalculateContentCRC32.mockReset(); - mockCalculateContentCRC32.mockResolvedValue(undefined); -}; const mockCalculateContentCRC32Reset = () => { mockCalculateContentCRC32.mockReset(); mockCalculateContentCRC32.mockImplementation( @@ -291,6 +296,9 @@ describe('getMultipartUploadHandlers with key', () => { const { multipartUploadJob } = getMultipartUploadHandlers({ key: defaultKey, data: twoPartsPayload, + options: { + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, + }, }); await multipartUploadJob(); @@ -301,9 +309,11 @@ describe('getMultipartUploadHandlers with key', () => { * * uploading each part calls calculateContentCRC32 1 time each * - * these steps results in 5 calls in total + * 1 time for optionsHash + * + * these steps results in 6 calls in total */ - expect(calculateContentCRC32).toHaveBeenCalledTimes(5); + expect(calculateContentCRC32).toHaveBeenCalledTimes(6); expect(calculateContentMd5).not.toHaveBeenCalled(); expect(mockUploadPart).toHaveBeenCalledTimes(2); expect(mockUploadPart).toHaveBeenCalledWith( @@ -317,8 +327,7 @@ describe('getMultipartUploadHandlers with key', () => { }, ); - it('should use md5 if crc32 is returning undefined', async () => { - mockCalculateContentCRC32Undefined(); + it('should use md5 if no using crc32', async () => { mockMultipartUploadSuccess(); Amplify.libraryOptions = { Storage: { @@ -372,6 +381,9 @@ describe('getMultipartUploadHandlers with key', () => { { key: defaultKey, data: file, + options: { + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, + }, }, file.size, ); @@ -589,7 +601,7 @@ describe('getMultipartUploadHandlers with key', () => { expect(Object.keys(cacheValue)).toEqual([ expect.stringMatching( // \d{13} is the file lastModified property of a file - /someName_\d{13}_8388608_application\/octet-stream_bucket_public_key/, + /someName_\d{13}_\/twwTw==_8388608_application\/octet-stream_bucket_public_key/, ), ]); }); @@ -800,7 +812,7 @@ describe('getMultipartUploadHandlers with key', () => { >; mockDefaultStorage.getItem.mockResolvedValue( JSON.stringify({ - [defaultCacheKey]: { + [generateDefaultCacheKey('o6a/Qw==')]: { uploadId: 'uploadId', bucket, key: defaultKey, @@ -942,6 +954,9 @@ describe('getMultipartUploadHandlers with path', () => { const { multipartUploadJob } = getMultipartUploadHandlers({ path: testPath, data: twoPartsPayload, + options: { + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, + }, }); await multipartUploadJob(); @@ -952,9 +967,11 @@ describe('getMultipartUploadHandlers with path', () => { * * uploading each part calls calculateContentCRC32 1 time each * - * these steps results in 5 calls in total + * 1 time for optionsHash + * + * these steps results in 6 calls in total */ - expect(calculateContentCRC32).toHaveBeenCalledTimes(5); + expect(calculateContentCRC32).toHaveBeenCalledTimes(6); expect(calculateContentMd5).not.toHaveBeenCalled(); expect(mockUploadPart).toHaveBeenCalledTimes(2); expect(mockUploadPart).toHaveBeenCalledWith( @@ -968,8 +985,7 @@ describe('getMultipartUploadHandlers with path', () => { }, ); - it('should use md5 if crc32 is returning undefined', async () => { - mockCalculateContentCRC32Undefined(); + it('should use md5 if no using crc32', async () => { mockMultipartUploadSuccess(); Amplify.libraryOptions = { Storage: { @@ -1023,6 +1039,9 @@ describe('getMultipartUploadHandlers with path', () => { { path: testPath, data: file, + options: { + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, + }, }, file.size, ); @@ -1533,9 +1552,10 @@ describe('getMultipartUploadHandlers with path', () => { const mockDefaultStorage = defaultStorage as jest.Mocked< typeof defaultStorage >; + mockDefaultStorage.getItem.mockResolvedValue( JSON.stringify({ - [testPathCacheKey]: { + [generateTestPathCacheKey('o6a/Qw==')]: { uploadId: 'uploadId', bucket, key: testPath, diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts index a76871e2435..bc99e69712d 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts @@ -15,6 +15,8 @@ import { calculateContentMd5 } from '../../../../../src/providers/s3/utils'; import * as CRC32 from '../../../../../src/providers/s3/utils/crc32'; import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob'; import '../testUtils'; +import { UploadDataChecksumAlgorithm } from '../../../../../src/providers/s3/types/options'; +import { CHECKSUM_ALGORITHM_CRC32 } from '../../../../../src/providers/s3/utils/constants'; global.Blob = BlobPolyfill as any; global.File = FilePolyfill as any; @@ -75,66 +77,77 @@ mockPutObject.mockResolvedValue({ /* TODO Remove suite when `key` parameter is removed */ describe('putObjectJob with key', () => { beforeEach(() => { + mockPutObject.mockClear(); jest.spyOn(CRC32, 'calculateContentCRC32').mockRestore(); }); - it('should supply the correct parameters to putObject API handler', async () => { - const abortController = new AbortController(); - const inputKey = 'key'; - const data = 'data'; - const mockContentType = 'contentType'; - const contentDisposition = 'contentDisposition'; - const contentEncoding = 'contentEncoding'; - const mockMetadata = { key: 'value' }; - const onProgress = jest.fn(); - const useAccelerateEndpoint = true; + it.each<{ checksumAlgorithm: UploadDataChecksumAlgorithm | undefined }>([ + { checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32 }, + { checksumAlgorithm: undefined }, + ])( + 'should supply the correct parameters to putObject API handler with checksumAlgorithm as $checksumAlgorithm', + async ({ checksumAlgorithm }) => { + const abortController = new AbortController(); + const inputKey = 'key'; + const data = 'data'; + const mockContentType = 'contentType'; + const contentDisposition = 'contentDisposition'; + const contentEncoding = 'contentEncoding'; + const mockMetadata = { key: 'value' }; + const onProgress = jest.fn(); + const useAccelerateEndpoint = true; - const job = putObjectJob( - { + const job = putObjectJob( + { + key: inputKey, + data, + options: { + contentDisposition, + contentEncoding, + contentType: mockContentType, + metadata: mockMetadata, + onProgress, + useAccelerateEndpoint, + checksumAlgorithm, + }, + }, + abortController.signal, + ); + const result = await job(); + expect(result).toEqual({ key: inputKey, - data, - options: { - contentDisposition, - contentEncoding, - contentType: mockContentType, - metadata: mockMetadata, - onProgress, - useAccelerateEndpoint, + eTag: 'eTag', + versionId: 'versionId', + contentType: 'contentType', + metadata: { key: 'value' }, + size: undefined, + }); + expect(mockPutObject).toHaveBeenCalledTimes(1); + await expect(mockPutObject).toBeLastCalledWithConfigAndInput( + { + credentials, + region, + abortSignal: abortController.signal, + onUploadProgress: expect.any(Function), + useAccelerateEndpoint: true, + userAgentValue: expect.any(String), }, - }, - abortController.signal, - ); - const result = await job(); - expect(result).toEqual({ - key: inputKey, - eTag: 'eTag', - versionId: 'versionId', - contentType: 'contentType', - metadata: { key: 'value' }, - size: undefined, - }); - expect(mockPutObject).toHaveBeenCalledTimes(1); - await expect(mockPutObject).toBeLastCalledWithConfigAndInput( - { - credentials, - region, - abortSignal: abortController.signal, - onUploadProgress: expect.any(Function), - useAccelerateEndpoint: true, - userAgentValue: expect.any(String), - }, - { - Bucket: bucket, - Key: `public/${inputKey}`, - Body: data, - ContentType: mockContentType, - ContentDisposition: contentDisposition, - ContentEncoding: contentEncoding, - Metadata: mockMetadata, - ChecksumCRC32: 'rfPzYw==', - }, - ); - }); + { + Bucket: bucket, + Key: `public/${inputKey}`, + Body: data, + ContentType: mockContentType, + ContentDisposition: contentDisposition, + ContentEncoding: contentEncoding, + Metadata: mockMetadata, + ChecksumCRC32: + checksumAlgorithm === CHECKSUM_ALGORITHM_CRC32 + ? 'rfPzYw==' + : undefined, + }, + ); + }, + ); it('should set ContentMD5 if object lock is enabled', async () => { jest @@ -193,7 +206,6 @@ describe('putObjectJob with key', () => { Key: 'public/key', Body: data, ContentType: 'application/octet-stream', - ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -225,7 +237,6 @@ describe('putObjectJob with key', () => { Key: 'public/key', Body: data, ContentType: 'application/octet-stream', - ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -238,18 +249,39 @@ describe('putObjectJob with path', () => { jest.spyOn(CRC32, 'calculateContentCRC32').mockRestore(); }); - test.each([ + it.each<{ checksumAlgorithm: UploadDataChecksumAlgorithm | undefined }>([ + { checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32 }, + { checksumAlgorithm: undefined }, + ]); + + test.each<{ + path: string | (() => string); + expectedKey: string; + checksumAlgorithm: UploadDataChecksumAlgorithm | undefined; + }>([ + { + path: testPath, + expectedKey: testPath, + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, + }, + { + path: () => testPath, + expectedKey: testPath, + checksumAlgorithm: CHECKSUM_ALGORITHM_CRC32, + }, { path: testPath, expectedKey: testPath, + checksumAlgorithm: undefined, }, { path: () => testPath, expectedKey: testPath, + checksumAlgorithm: undefined, }, ])( - 'should supply the correct parameters to putObject API handler when path is $path', - async ({ path: inputPath, expectedKey }) => { + 'should supply the correct parameters to putObject API handler when path is $path and checksumAlgorithm is $checksumAlgorithm', + async ({ path: inputPath, expectedKey, checksumAlgorithm }) => { const abortController = new AbortController(); const data = 'data'; const mockContentType = 'contentType'; @@ -270,6 +302,7 @@ describe('putObjectJob with path', () => { metadata: mockMetadata, onProgress, useAccelerateEndpoint, + checksumAlgorithm, }, }, abortController.signal, @@ -301,7 +334,10 @@ describe('putObjectJob with path', () => { ContentDisposition: contentDisposition, ContentEncoding: contentEncoding, Metadata: mockMetadata, - ChecksumCRC32: 'rfPzYw==', + ChecksumCRC32: + checksumAlgorithm === CHECKSUM_ALGORITHM_CRC32 + ? 'rfPzYw==' + : undefined, }, ); }, @@ -439,7 +475,6 @@ describe('putObjectJob with path', () => { Key: 'path/', Body: data, ContentType: 'application/octet-stream', - ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -471,7 +506,6 @@ describe('putObjectJob with path', () => { Key: 'path/', Body: data, ContentType: 'application/octet-stream', - ChecksumCRC32: 'rfPzYw==', }, ); }); diff --git a/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts b/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts index 0f4c1adce27..e82080345d7 100644 --- a/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts +++ b/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts @@ -1,13 +1,131 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { + TextDecoder as TextDecoderPolyfill, + TextEncoder as TextEncoderPolyfill, +} from 'node:util'; + import { calculateContentCRC32 } from '../../../../src/providers/s3/utils/crc32.native'; +global.TextEncoder = TextEncoderPolyfill as any; +global.TextDecoder = TextDecoderPolyfill as any; + const MB = 1024 * 1024; const getBlob = (size: number) => new Blob(['1'.repeat(size)]); +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); -describe('calculate crc32 native', () => { - it('should return undefined', async () => { - expect(await calculateContentCRC32(getBlob(8 * MB))).toEqual(undefined); +describe('calculate crc32', () => { + describe.each([ + { + type: 'file', + size: '4B', + data: new File(['data'], 'someName'), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'blob', + size: '4B', + data: new Blob(['data']), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'string', + size: '4B', + data: 'data', + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'arrayBuffer', + size: '4B', + data: new Uint8Array(encoder.encode('data')).buffer, + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'arrayBufferView', + size: '4B', + data: new DataView(encoder.encode('1234 data 5678').buffer, 5, 4), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'file', + size: '8MB', + data: new File([getBlob(8 * MB)], 'someName'), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'blob', + size: '8MB', + data: getBlob(8 * MB), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'string', + size: '8MB', + data: '1'.repeat(8 * MB), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'arrayBuffer', + size: '8MB', + data: new Uint8Array(encoder.encode('1'.repeat(8 * MB))).buffer, + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'arrayBufferView', + size: '8MB', + data: encoder.encode('1'.repeat(8 * MB)), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + ])('output for data type of $type with size $size', ({ data, expected }) => { + it('should match expected checksum results', async () => { + const result = (await calculateContentCRC32(data))!; + expect(result.checksum).toEqual(expected.checksum); + expect(result.seed).toEqual(expected.seed); + expect(decoder.decode(result.checksumArrayBuffer)).toEqual( + decoder.decode(expected.checksumArrayBuffer), + ); + }); }); }); diff --git a/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts b/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts new file mode 100644 index 00000000000..875c1f90466 --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts @@ -0,0 +1,113 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Blob as BlobPolyfill, File as FilePolyfill } from 'node:buffer'; +import { WritableStream as WritableStreamPolyfill } from 'node:stream/web'; +import { + TextDecoder as TextDecoderPolyfill, + TextEncoder as TextEncoderPolyfill, +} from 'node:util'; + +import { getCombinedCrc32 } from '../../../../src/providers/s3/utils/getCombinedCrc32.native'; +import { byteLength } from '../../../../src/providers/s3/apis/uploadData/byteLength'; + +global.Blob = BlobPolyfill as any; +global.File = FilePolyfill as any; +global.WritableStream = WritableStreamPolyfill as any; +global.TextEncoder = TextEncoderPolyfill as any; +global.TextDecoder = TextDecoderPolyfill as any; + +const MB = 1024 * 1024; +const getBlob = (size: number) => new Blob(['1'.repeat(size)]); +const encoder = new TextEncoder(); + +describe('calculate crc32', () => { + describe.each([ + { + type: 'file', + size: '4B', + data: new File(['data'], 'someName'), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'blob', + size: '4B', + data: new Blob(['data']), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'string', + size: '4B', + data: 'data', + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'arrayBuffer', + size: '4B', + data: new Uint8Array(encoder.encode('data')).buffer, + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'arrayBufferView', + size: '4B', + data: new DataView(encoder.encode('1234 data 5678').buffer, 5, 4), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'file', + size: '8MB', + data: new File([getBlob(8 * MB)], 'someName'), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'blob', + size: '8MB', + data: getBlob(8 * MB), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'string', + size: '8MB', + data: '1'.repeat(8 * MB), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'arrayBuffer', + size: '8MB', + data: new Uint8Array(encoder.encode('1'.repeat(8 * MB))).buffer, + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'arrayBufferView', + size: '8MB', + data: encoder.encode('1'.repeat(8 * MB)), + expected: { + checksum: 'hwOICA==-2', + }, + }, + ])('output for data type of $type with size $size', ({ data, expected }) => { + it('should match expected checksum results', async () => { + expect((await getCombinedCrc32(data, byteLength(data)))!).toEqual( + expected.checksum, + ); + }); + }); +}); diff --git a/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.test.ts b/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.test.ts new file mode 100644 index 00000000000..13ee357cd4a --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.test.ts @@ -0,0 +1,113 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Blob as BlobPolyfill, File as FilePolyfill } from 'node:buffer'; +import { WritableStream as WritableStreamPolyfill } from 'node:stream/web'; +import { + TextDecoder as TextDecoderPolyfill, + TextEncoder as TextEncoderPolyfill, +} from 'node:util'; + +import { getCombinedCrc32 } from '../../../../src/providers/s3/utils/getCombinedCrc32'; +import { byteLength } from '../../../../src/providers/s3/apis/uploadData/byteLength'; + +global.Blob = BlobPolyfill as any; +global.File = FilePolyfill as any; +global.WritableStream = WritableStreamPolyfill as any; +global.TextEncoder = TextEncoderPolyfill as any; +global.TextDecoder = TextDecoderPolyfill as any; + +const MB = 1024 * 1024; +const getBlob = (size: number) => new Blob(['1'.repeat(size)]); +const encoder = new TextEncoder(); + +describe('calculate crc32', () => { + describe.each([ + { + type: 'file', + size: '4B', + data: new File(['data'], 'someName'), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'blob', + size: '4B', + data: new Blob(['data']), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'string', + size: '4B', + data: 'data', + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'arrayBuffer', + size: '4B', + data: new Uint8Array(encoder.encode('data')).buffer, + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'arrayBufferView', + size: '4B', + data: new DataView(encoder.encode('1234 data 5678').buffer, 5, 4), + expected: { + checksum: 'wu1R0Q==-1', + }, + }, + { + type: 'file', + size: '8MB', + data: new File([getBlob(8 * MB)], 'someName'), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'blob', + size: '8MB', + data: getBlob(8 * MB), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'string', + size: '8MB', + data: '1'.repeat(8 * MB), + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'arrayBuffer', + size: '8MB', + data: new Uint8Array(encoder.encode('1'.repeat(8 * MB))).buffer, + expected: { + checksum: 'hwOICA==-2', + }, + }, + { + type: 'arrayBufferView', + size: '8MB', + data: encoder.encode('1'.repeat(8 * MB)), + expected: { + checksum: 'hwOICA==-2', + }, + }, + ])('output for data type of $type with size $size', ({ data, expected }) => { + it('should match expected checksum results', async () => { + expect((await getCombinedCrc32(data, byteLength(data)))!).toEqual( + expected.checksum, + ); + }); + }); +}); diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts index 32462a83545..8a35940abb6 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts @@ -3,19 +3,23 @@ import { StorageAccessLevel } from '@aws-amplify/core'; -import { ContentDisposition, ResolvedS3Config } from '../../../types/options'; +import { + ContentDisposition, + ResolvedS3Config, + UploadDataChecksumAlgorithm, +} from '../../../types/options'; import { StorageUploadDataPayload } from '../../../../../types'; import { Part, createMultipartUpload } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; -import { calculateContentCRC32 } from '../../../utils/crc32'; import { constructContentDisposition } from '../../../utils/constructContentDisposition'; +import { getCombinedCrc32 } from '../../../utils/getCombinedCrc32'; +import { CHECKSUM_ALGORITHM_CRC32 } from '../../../utils/constants'; import { cacheMultipartUpload, findCachedUploadParts, getUploadsCacheKey, } from './uploadCache'; -import { getDataChunker } from './getDataChunker'; interface LoadOrCreateMultipartUploadOptions { s3Config: ResolvedS3Config; @@ -30,6 +34,8 @@ interface LoadOrCreateMultipartUploadOptions { metadata?: Record; size?: number; abortSignal?: AbortSignal; + checksumAlgorithm?: UploadDataChecksumAlgorithm; + optionsHash: string; } interface LoadOrCreateMultipartUploadResult { @@ -57,6 +63,8 @@ export const loadOrCreateMultipartUpload = async ({ contentEncoding, metadata, abortSignal, + checksumAlgorithm, + optionsHash, }: LoadOrCreateMultipartUploadOptions): Promise => { const finalKey = keyPrefix !== undefined ? keyPrefix + key : key; @@ -79,6 +87,7 @@ export const loadOrCreateMultipartUpload = async ({ bucket, accessLevel, key, + optionsHash, }); const cachedUploadParts = await findCachedUploadParts({ @@ -99,7 +108,10 @@ export const loadOrCreateMultipartUpload = async ({ finalCrc32: cachedUpload.finalCrc32, }; } else { - const finalCrc32 = await getCombinedCrc32(data, size); + const finalCrc32 = + checksumAlgorithm === CHECKSUM_ALGORITHM_CRC32 + ? await getCombinedCrc32(data, size) + : undefined; const { UploadId } = await createMultipartUpload( { @@ -133,6 +145,7 @@ export const loadOrCreateMultipartUpload = async ({ bucket, accessLevel, key, + optionsHash, }); await cacheMultipartUpload(uploadCacheKey, { uploadId: UploadId!, @@ -149,20 +162,3 @@ export const loadOrCreateMultipartUpload = async ({ }; } }; - -const getCombinedCrc32 = async ( - data: StorageUploadDataPayload, - size: number | undefined, -) => { - const crc32List: ArrayBuffer[] = []; - const dataChunker = getDataChunker(data, size); - for (const { data: checkData } of dataChunker) { - const checksumArrayBuffer = (await calculateContentCRC32(checkData)) - ?.checksumArrayBuffer; - if (checksumArrayBuffer === undefined) return undefined; - - crc32List.push(checksumArrayBuffer); - } - - return `${(await calculateContentCRC32(new Blob(crc32List)))?.checksum}-${crc32List.length}`; -}; diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts index 9761ee85732..10d7c0ddd5a 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts @@ -100,6 +100,7 @@ interface UploadsCacheKeyOptions { accessLevel?: StorageAccessLevel; key: string; file?: File; + optionsHash: string; } /** @@ -114,6 +115,7 @@ export const getUploadsCacheKey = ({ bucket, accessLevel, key, + optionsHash, }: UploadsCacheKeyOptions) => { let levelStr; const resolvedContentType = @@ -126,7 +128,7 @@ export const getUploadsCacheKey = ({ levelStr = accessLevel === 'guest' ? 'public' : accessLevel; } - const baseId = `${size}_${resolvedContentType}_${bucket}_${levelStr}_${key}`; + const baseId = `${optionsHash}_${size}_${resolvedContentType}_${bucket}_${levelStr}_${key}`; if (file) { return `${file.name}_${file.lastModified}_${baseId}`; diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts index 4fafe7100a9..70c28d5cd80 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts @@ -30,6 +30,7 @@ import { import { getStorageUserAgentValue } from '../../../utils/userAgent'; import { logger } from '../../../../../utils'; import { validateObjectNotExists } from '../validateObjectNotExists'; +import { calculateContentCRC32 } from '../../../utils/crc32'; import { uploadPartExecutor } from './uploadPartExecutor'; import { getUploadsCacheKey, removeCachedUpload } from './uploadCache'; @@ -110,6 +111,10 @@ export const getMultipartUploadHandlers = ( resolvedAccessLevel = resolveAccessLevel(accessLevel); } + const optionsHash = ( + await calculateContentCRC32(JSON.stringify(uploadDataOptions)) + ).checksum; + if (!inProgressUpload) { const { uploadId, cachedParts, finalCrc32 } = await loadOrCreateMultipartUpload({ @@ -125,6 +130,8 @@ export const getMultipartUploadHandlers = ( data, size, abortSignal: abortController.signal, + checksumAlgorithm: uploadDataOptions?.checksumAlgorithm, + optionsHash, }); inProgressUpload = { uploadId, @@ -141,6 +148,7 @@ export const getMultipartUploadHandlers = ( bucket: resolvedBucket!, size, key: objectKey, + optionsHash, }) : undefined; diff --git a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts index cbd602580a0..f29ccf12ec6 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts @@ -13,7 +13,10 @@ import { import { ItemWithKey, ItemWithPath } from '../../types/outputs'; import { putObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; -import { STORAGE_INPUT_KEY } from '../../utils/constants'; +import { + CHECKSUM_ALGORITHM_CRC32, + STORAGE_INPUT_KEY, +} from '../../utils/constants'; import { calculateContentCRC32 } from '../../utils/crc32'; import { constructContentDisposition } from '../../utils/constructContentDisposition'; @@ -47,10 +50,15 @@ export const putObjectJob = contentType = 'application/octet-stream', preventOverwrite, metadata, + checksumAlgorithm, onProgress, } = uploadDataOptions ?? {}; - const checksumCRC32 = await calculateContentCRC32(data); + const checksumCRC32 = + checksumAlgorithm === CHECKSUM_ALGORITHM_CRC32 + ? await calculateContentCRC32(data) + : undefined; + const contentMD5 = // check if checksum exists. ex: should not exist in react native !checksumCRC32 && isObjectLockEnabled diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index 3442b6867cc..badb9267d39 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -192,6 +192,8 @@ export type DownloadDataOptions = CommonOptions & export type DownloadDataWithKeyOptions = ReadOptions & DownloadDataOptions; export type DownloadDataWithPathOptions = DownloadDataOptions; +export type UploadDataChecksumAlgorithm = 'crc-32'; + export type UploadDataOptions = CommonOptions & TransferOptions & { /** @@ -222,6 +224,12 @@ export type UploadDataOptions = CommonOptions & * @default false */ preventOverwrite?: boolean; + /** + * The algorithm used to compute a checksum for the object. Used to verify that the data received by S3 + * matches what was originally sent. Disabled by default. + * @default undefined + */ + checksumAlgorithm?: UploadDataChecksumAlgorithm; }; /** @deprecated Use {@link UploadDataWithPathOptions} instead. */ diff --git a/packages/storage/src/providers/s3/utils/constants.ts b/packages/storage/src/providers/s3/utils/constants.ts index 538f3a902ff..72a58b778de 100644 --- a/packages/storage/src/providers/s3/utils/constants.ts +++ b/packages/storage/src/providers/s3/utils/constants.ts @@ -28,3 +28,5 @@ export const STORAGE_INPUT_KEY = 'key'; export const STORAGE_INPUT_PATH = 'path'; export const DEFAULT_DELIMITER = '/'; + +export const CHECKSUM_ALGORITHM_CRC32 = 'crc-32'; diff --git a/packages/storage/src/providers/s3/utils/crc32.native.ts b/packages/storage/src/providers/s3/utils/crc32.native.ts index 389cb5fc87b..3f906eda0c8 100644 --- a/packages/storage/src/providers/s3/utils/crc32.native.ts +++ b/packages/storage/src/providers/s3/utils/crc32.native.ts @@ -1,11 +1,77 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { CRC32Checksum } from './crc32'; +import crc32 from 'crc-32'; + +import { hexToArrayBuffer, hexToBase64 } from './hexUtils'; + +const CHUNK_SIZE = 1024 * 1024; // 1MB chunks + +export interface CRC32Checksum { + checksumArrayBuffer: ArrayBuffer; + checksum: string; + seed: number; +} export const calculateContentCRC32 = async ( content: Blob | string | ArrayBuffer | ArrayBufferView, - _seed = 0, -): Promise => { - return undefined; + seed = 0, +): Promise => { + let internalSeed = seed; + + if (content instanceof ArrayBuffer || ArrayBuffer.isView(content)) { + let uint8Array: Uint8Array; + + if (content instanceof ArrayBuffer) { + uint8Array = new Uint8Array(content); + } else { + uint8Array = new Uint8Array( + content.buffer, + content.byteOffset, + content.byteLength, + ); + } + + let offset = 0; + while (offset < uint8Array.length) { + const end = Math.min(offset + CHUNK_SIZE, uint8Array.length); + const chunk = uint8Array.slice(offset, end); + internalSeed = crc32.buf(chunk, internalSeed) >>> 0; + offset = end; + } + } else { + let blob: Blob; + + if (content instanceof Blob) { + blob = content; + } else { + blob = new Blob([content]); + } + + let offset = 0; + while (offset < blob.size) { + const end = Math.min(offset + CHUNK_SIZE, blob.size); + const chunk = blob.slice(offset, end); + const arrayBuffer = await new Promise(resolve => { + const reader = new FileReader(); + reader.onload = () => { + resolve(reader.result as ArrayBuffer); + }; + reader.readAsArrayBuffer(chunk); + }); + const uint8Array = new Uint8Array(arrayBuffer); + + internalSeed = crc32.buf(uint8Array, internalSeed) >>> 0; + + offset = end; + } + } + + const hex = internalSeed.toString(16).padStart(8, '0'); + + return { + checksumArrayBuffer: hexToArrayBuffer(hex), + checksum: hexToBase64(hex), + seed: internalSeed, + }; }; diff --git a/packages/storage/src/providers/s3/utils/crc32.ts b/packages/storage/src/providers/s3/utils/crc32.ts index 6d9e194c3af..20e7247f593 100644 --- a/packages/storage/src/providers/s3/utils/crc32.ts +++ b/packages/storage/src/providers/s3/utils/crc32.ts @@ -3,6 +3,8 @@ import crc32 from 'crc-32'; +import { hexToArrayBuffer, hexToBase64 } from './hexUtils'; + export interface CRC32Checksum { checksumArrayBuffer: ArrayBuffer; checksum: string; @@ -12,7 +14,7 @@ export interface CRC32Checksum { export const calculateContentCRC32 = async ( content: Blob | string | ArrayBuffer | ArrayBufferView, seed = 0, -): Promise => { +): Promise => { let internalSeed = seed; let blob: Blob; @@ -37,15 +39,3 @@ export const calculateContentCRC32 = async ( seed: internalSeed, }; }; - -const hexToArrayBuffer = (hexString: string) => - new Uint8Array((hexString.match(/\w{2}/g)! ?? []).map(h => parseInt(h, 16))) - .buffer; - -const hexToBase64 = (hexString: string) => - btoa( - hexString - .match(/\w{2}/g)! - .map((a: string) => String.fromCharCode(parseInt(a, 16))) - .join(''), - ); diff --git a/packages/storage/src/providers/s3/utils/getCombinedCrc32.native.ts b/packages/storage/src/providers/s3/utils/getCombinedCrc32.native.ts new file mode 100644 index 00000000000..f33a8689f4a --- /dev/null +++ b/packages/storage/src/providers/s3/utils/getCombinedCrc32.native.ts @@ -0,0 +1,46 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageUploadDataPayload } from '../../../types'; +import { getDataChunker } from '../apis/uploadData/multipart/getDataChunker'; + +import { calculateContentCRC32 } from './crc32'; + +/** + * Calculates a combined CRC32 checksum for the given data. + * + * This function chunks the input data, calculates CRC32 for each chunk, + * and then combines these checksums into a single value. + * + * @async + * @param {StorageUploadDataPayload} data - The data to calculate the checksum for. + * @param {number | undefined} size - The size of each chunk. If undefined, a default chunk size will be used. + * @returns {Promise} A promise that resolves to a string containing the combined CRC32 checksum + * and the number of chunks, separated by a hyphen. + */ +export const getCombinedCrc32 = async ( + data: StorageUploadDataPayload, + size: number | undefined, +) => { + const crc32List: Uint8Array[] = []; + const dataChunker = getDataChunker(data, size); + + let totalLength = 0; + for (const { data: checkData } of dataChunker) { + const checksum = new Uint8Array( + (await calculateContentCRC32(checkData)).checksumArrayBuffer, + ); + totalLength += checksum.length; + crc32List.push(checksum); + } + + // Combine all Uint8Arrays into a single Uint8Array + const combinedArray = new Uint8Array(totalLength); + let offset = 0; + for (const crc32Hash of crc32List) { + combinedArray.set(crc32Hash, offset); + offset += crc32Hash.length; + } + + return `${(await calculateContentCRC32(combinedArray.buffer)).checksum}-${crc32List.length}`; +}; diff --git a/packages/storage/src/providers/s3/utils/getCombinedCrc32.ts b/packages/storage/src/providers/s3/utils/getCombinedCrc32.ts new file mode 100644 index 00000000000..c7e927b381d --- /dev/null +++ b/packages/storage/src/providers/s3/utils/getCombinedCrc32.ts @@ -0,0 +1,34 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageUploadDataPayload } from '../../../types'; +import { getDataChunker } from '../apis/uploadData/multipart/getDataChunker'; + +import { calculateContentCRC32 } from './crc32'; + +/** + * Calculates a combined CRC32 checksum for the given data. + * + * This function chunks the input data, calculates CRC32 for each chunk, + * and then combines these checksums into a single value. + * + * @async + * @param {StorageUploadDataPayload} data - The data to calculate the checksum for. + * @param {number | undefined} size - The size of each chunk. If undefined, a default chunk size will be used. + * @returns {Promise} A promise that resolves to a string containing the combined CRC32 checksum + * and the number of chunks, separated by a hyphen. + */ +export const getCombinedCrc32 = async ( + data: StorageUploadDataPayload, + size: number | undefined, +) => { + const crc32List: ArrayBuffer[] = []; + const dataChunker = getDataChunker(data, size); + for (const { data: checkData } of dataChunker) { + const { checksumArrayBuffer } = await calculateContentCRC32(checkData); + + crc32List.push(checksumArrayBuffer); + } + + return `${(await calculateContentCRC32(new Blob(crc32List))).checksum}-${crc32List.length}`; +}; diff --git a/packages/storage/src/providers/s3/utils/hexUtils.ts b/packages/storage/src/providers/s3/utils/hexUtils.ts new file mode 100644 index 00000000000..a71f94e9f98 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/hexUtils.ts @@ -0,0 +1,14 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const hexToArrayBuffer = (hexString: string) => + new Uint8Array((hexString.match(/\w{2}/g)! ?? []).map(h => parseInt(h, 16))) + .buffer; + +export const hexToBase64 = (hexString: string) => + btoa( + hexString + .match(/\w{2}/g)! + .map((a: string) => String.fromCharCode(parseInt(a, 16))) + .join(''), + );