From 51450645a9d6a0aef93144cdda42a5bf07cf9ebe Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Thu, 17 Oct 2024 13:43:45 -0700 Subject: [PATCH] feat(storage): add advanced option to disable upload cache (#13931) --- packages/aws-amplify/package.json | 2 +- .../s3/apis/uploadData/index.test.ts | 34 ++++++- .../apis/uploadData/multipartHandlers.test.ts | 93 +++++++++++++++++-- .../storage/src/internals/types/inputs.ts | 6 +- .../s3/apis/internal/uploadData/index.ts | 12 +-- .../internal/uploadData/multipart/index.ts | 5 +- .../uploadData/multipart/initialUpload.ts | 23 +++-- .../uploadData/multipart/uploadCache.ts | 29 +++--- .../uploadData/multipart/uploadHandlers.ts | 52 +++++++++-- .../apis/internal/uploadData/putObjectJob.ts | 17 +++- .../src/providers/s3/apis/uploadData.ts | 12 ++- 11 files changed, 231 insertions(+), 54 deletions(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index efcf9151416..a629f3d5a51 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "22.37 kB" + "limit": "22.39 kB" } ] } diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts index c967374f17e..01a245a9ae1 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts @@ -1,6 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { defaultStorage } from '@aws-amplify/core'; + import { uploadData } from '../../../../../src/providers/s3/apis'; import { MAX_OBJECT_SIZE } from '../../../../../src/providers/s3/utils/constants'; import { createUploadTask } from '../../../../../src/providers/s3/utils'; @@ -70,6 +72,22 @@ describe('uploadData with key', () => { expect(mockGetMultipartUploadHandlers).not.toHaveBeenCalled(); }); + it('should use putObject for 0 bytes data (e.g. create a folder)', () => { + const testInput = { + key: 'key', + data: '', // 0 bytes + }; + + uploadData(testInput); + + expect(mockPutObjectJob).toHaveBeenCalledWith( + expect.objectContaining(testInput), + expect.any(AbortSignal), + expect.any(Number), + ); + expect(mockGetMultipartUploadHandlers).not.toHaveBeenCalled(); + }); + it('should use uploadTask', async () => { mockPutObjectJob.mockReturnValueOnce('putObjectJob'); mockCreateUploadTask.mockReturnValueOnce('uploadTask'); @@ -175,7 +193,7 @@ describe('uploadData with path', () => { uploadData(testInput); expect(mockPutObjectJob).toHaveBeenCalledWith( - testInput, + expect.objectContaining(testInput), expect.any(AbortSignal), expect.any(Number), ); @@ -192,7 +210,7 @@ describe('uploadData with path', () => { uploadData(testInput); expect(mockPutObjectJob).toHaveBeenCalledWith( - testInput, + expect.objectContaining(testInput), expect.any(AbortSignal), expect.any(Number), ); @@ -231,7 +249,12 @@ describe('uploadData with path', () => { expect(mockPutObjectJob).not.toHaveBeenCalled(); expect(mockGetMultipartUploadHandlers).toHaveBeenCalledWith( - testInput, + expect.objectContaining({ + ...testInput, + options: { + resumableUploadsCache: defaultStorage, + }, + }), expect.any(Number), ); }); @@ -291,9 +314,10 @@ describe('uploadData with path', () => { }; uploadData(testInput); expect(mockGetMultipartUploadHandlers).toHaveBeenCalledWith( - expect.objectContaining({ + { ...testInput, - }), + options: expect.objectContaining(testInput.options), + }, expect.any(Number), ); diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts index 347ef12b0aa..06771dce52e 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts @@ -524,6 +524,23 @@ describe('getMultipartUploadHandlers with key', () => { mockDefaultStorage.setItem.mockReset(); }); + it('should disable upload caching if resumableUploadsCache option is not set', async () => { + mockMultipartUploadSuccess(); + const size = 8 * MB; + const { multipartUploadJob } = getMultipartUploadHandlers( + { + key: defaultKey, + data: new ArrayBuffer(size), + }, + size, + ); + await multipartUploadJob(); + expect(mockDefaultStorage.getItem).not.toHaveBeenCalled(); + expect(mockDefaultStorage.setItem).not.toHaveBeenCalled(); + expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); + expect(mockListParts).not.toHaveBeenCalled(); + }); + it('should send createMultipartUpload request if the upload task is not cached', async () => { mockMultipartUploadSuccess(); const size = 8 * MB; @@ -531,6 +548,9 @@ describe('getMultipartUploadHandlers with key', () => { { key: defaultKey, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -559,6 +579,9 @@ describe('getMultipartUploadHandlers with key', () => { { key: defaultKey, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -577,6 +600,9 @@ describe('getMultipartUploadHandlers with key', () => { { key: defaultKey, data: new File([new ArrayBuffer(size)], 'someName'), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -612,6 +638,9 @@ describe('getMultipartUploadHandlers with key', () => { { key: defaultKey, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -630,6 +659,9 @@ describe('getMultipartUploadHandlers with key', () => { { key: defaultKey, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -657,6 +689,9 @@ describe('getMultipartUploadHandlers with key', () => { { key: defaultKey, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -679,6 +714,9 @@ describe('getMultipartUploadHandlers with key', () => { { key: defaultKey, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -795,9 +833,7 @@ describe('getMultipartUploadHandlers with key', () => { it('should send progress for cached upload parts', async () => { mockMultipartUploadSuccess(); - const mockDefaultStorage = defaultStorage as jest.Mocked< - typeof defaultStorage - >; + const mockDefaultStorage = jest.mocked(defaultStorage); mockDefaultStorage.getItem.mockResolvedValue( JSON.stringify({ [defaultCacheKey]: { @@ -819,6 +855,7 @@ describe('getMultipartUploadHandlers with key', () => { data: new ArrayBuffer(8 * MB), options: { onProgress, + resumableUploadsCache: mockDefaultStorage, }, }, 8 * MB, @@ -1258,6 +1295,23 @@ describe('getMultipartUploadHandlers with path', () => { mockDefaultStorage.setItem.mockReset(); }); + it('should disable upload caching if resumableUploadsCache option is not set', async () => { + mockMultipartUploadSuccess(); + const size = 8 * MB; + const { multipartUploadJob } = getMultipartUploadHandlers( + { + key: defaultKey, + data: new ArrayBuffer(size), + }, + size, + ); + await multipartUploadJob(); + expect(mockDefaultStorage.getItem).not.toHaveBeenCalled(); + expect(mockDefaultStorage.setItem).not.toHaveBeenCalled(); + expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); + expect(mockListParts).not.toHaveBeenCalled(); + }); + it('should send createMultipartUpload request if the upload task is not cached', async () => { mockMultipartUploadSuccess(); const size = 8 * MB; @@ -1265,6 +1319,9 @@ describe('getMultipartUploadHandlers with path', () => { { path: testPath, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -1293,6 +1350,9 @@ describe('getMultipartUploadHandlers with path', () => { { path: testPath, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -1311,6 +1371,9 @@ describe('getMultipartUploadHandlers with path', () => { { path: testPath, data: new File([new ArrayBuffer(size)], 'someName'), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -1321,12 +1384,10 @@ describe('getMultipartUploadHandlers with path', () => { mockDefaultStorage.setItem.mock.calls[0][1], ); - // \d{13} is the file lastModified property of a file - const lastModifiedRegex = /someName_\d{13}_/; - expect(Object.keys(cacheValue)).toEqual([ expect.stringMatching( - new RegExp(lastModifiedRegex.source + testPathCacheKey), + // \d{13} is the file lastModified property of a file + new RegExp('someName_\\d{13}_' + testPathCacheKey), ), ]); }); @@ -1349,6 +1410,9 @@ describe('getMultipartUploadHandlers with path', () => { { path: testPath, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -1367,6 +1431,9 @@ describe('getMultipartUploadHandlers with path', () => { { path: testPath, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -1392,6 +1459,9 @@ describe('getMultipartUploadHandlers with path', () => { { path: testPath, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -1414,6 +1484,9 @@ describe('getMultipartUploadHandlers with path', () => { { path: testPath, data: new ArrayBuffer(size), + options: { + resumableUploadsCache: mockDefaultStorage, + }, }, size, ); @@ -1530,9 +1603,8 @@ describe('getMultipartUploadHandlers with path', () => { it('should send progress for cached upload parts', async () => { mockMultipartUploadSuccess(); - const mockDefaultStorage = defaultStorage as jest.Mocked< - typeof defaultStorage - >; + const mockDefaultStorage = jest.mocked(defaultStorage); + mockDefaultStorage.getItem.mockResolvedValue( JSON.stringify({ [testPathCacheKey]: { @@ -1554,6 +1626,7 @@ describe('getMultipartUploadHandlers with path', () => { data: new ArrayBuffer(8 * MB), options: { onProgress, + resumableUploadsCache: mockDefaultStorage, }, }, 8 * MB, diff --git a/packages/storage/src/internals/types/inputs.ts b/packages/storage/src/internals/types/inputs.ts index 7d3d6ca94bb..94c92ee849b 100644 --- a/packages/storage/src/internals/types/inputs.ts +++ b/packages/storage/src/internals/types/inputs.ts @@ -11,13 +11,11 @@ import { DownloadDataWithPathInput, GetPropertiesWithPathInput, GetUrlWithPathInput, + ListAllWithPathInput, + ListPaginateWithPathInput, RemoveWithPathInput, UploadDataWithPathInput, } from '../../providers/s3'; -import { - ListAllWithPathInput, - ListPaginateWithPathInput, -} from '../../providers/s3/types/inputs'; import { CredentialsProvider, ListLocationsInput } from './credentials'; import { Permission, PrefixType, Privilege } from './common'; diff --git a/packages/storage/src/providers/s3/apis/internal/uploadData/index.ts b/packages/storage/src/providers/s3/apis/internal/uploadData/index.ts index ee211c80347..a31aff1c9c6 100644 --- a/packages/storage/src/providers/s3/apis/internal/uploadData/index.ts +++ b/packages/storage/src/providers/s3/apis/internal/uploadData/index.ts @@ -1,20 +1,20 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { UploadDataInput } from '../../../types'; -// TODO: Remove this interface when we move to public advanced APIs. -import { UploadDataInput as UploadDataWithPathInputWithAdvancedOptions } from '../../../../../internals/types/inputs'; import { createUploadTask } from '../../../utils'; import { assertValidationError } from '../../../../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../../../../errors/types/validation'; import { DEFAULT_PART_SIZE, MAX_OBJECT_SIZE } from '../../../utils/constants'; import { byteLength } from './byteLength'; -import { putObjectJob } from './putObjectJob'; -import { getMultipartUploadHandlers } from './multipart'; +import { SinglePartUploadDataInput, putObjectJob } from './putObjectJob'; +import { + MultipartUploadDataInput, + getMultipartUploadHandlers, +} from './multipart'; export const uploadData = ( - input: UploadDataInput | UploadDataWithPathInputWithAdvancedOptions, + input: SinglePartUploadDataInput | MultipartUploadDataInput, ) => { const { data } = input; diff --git a/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/index.ts b/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/index.ts index 1f7db0aabb3..576f715dd79 100644 --- a/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/index.ts +++ b/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/index.ts @@ -1,4 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -export { getMultipartUploadHandlers } from './uploadHandlers'; +export { + getMultipartUploadHandlers, + MultipartUploadDataInput, +} from './uploadHandlers'; diff --git a/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/initialUpload.ts b/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/initialUpload.ts index 0d944831994..ff7c181f7fe 100644 --- a/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/initialUpload.ts +++ b/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/initialUpload.ts @@ -1,7 +1,10 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { StorageAccessLevel } from '@aws-amplify/core'; +import { + KeyValueStorageInterface, + StorageAccessLevel, +} from '@aws-amplify/core'; import { ContentDisposition, @@ -33,6 +36,7 @@ interface LoadOrCreateMultipartUploadOptions { metadata?: Record; size?: number; abortSignal?: AbortSignal; + resumableUploadsCache?: KeyValueStorageInterface; expectedBucketOwner?: string; } @@ -61,6 +65,7 @@ export const loadOrCreateMultipartUpload = async ({ contentEncoding, metadata, abortSignal, + resumableUploadsCache, expectedBucketOwner, }: LoadOrCreateMultipartUploadOptions): Promise => { const finalKey = keyPrefix !== undefined ? keyPrefix + key : key; @@ -73,8 +78,11 @@ export const loadOrCreateMultipartUpload = async ({ finalCrc32?: string; } | undefined; - if (size === undefined) { - logger.debug('uploaded data size cannot be determined, skipping cache.'); + + if (size === undefined || !resumableUploadsCache) { + logger.debug( + 'uploaded data size or cache instance cannot be determined, skipping cache.', + ); cachedUpload = undefined; } else { const uploadCacheKey = getUploadsCacheKey({ @@ -91,6 +99,7 @@ export const loadOrCreateMultipartUpload = async ({ cacheKey: uploadCacheKey, bucket, finalKey, + resumableUploadsCache, }); cachedUpload = cachedUploadParts ? { ...cachedUploadParts, uploadCacheKey } @@ -123,8 +132,10 @@ export const loadOrCreateMultipartUpload = async ({ }, ); - if (size === undefined) { - logger.debug('uploaded data size cannot be determined, skipping cache.'); + if (size === undefined || !resumableUploadsCache) { + logger.debug( + 'uploaded data size or cache instance cannot be determined, skipping cache.', + ); return { uploadId: UploadId!, @@ -140,7 +151,7 @@ export const loadOrCreateMultipartUpload = async ({ accessLevel, key, }); - await cacheMultipartUpload(uploadCacheKey, { + await cacheMultipartUpload(resumableUploadsCache, uploadCacheKey, { uploadId: UploadId!, bucket, key, diff --git a/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/uploadCache.ts b/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/uploadCache.ts index 998e750cc9b..6c05a967d7b 100644 --- a/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/uploadCache.ts +++ b/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/uploadCache.ts @@ -4,7 +4,6 @@ import { KeyValueStorageInterface, StorageAccessLevel, - defaultStorage, } from '@aws-amplify/core'; import { UPLOADS_STORAGE_KEY } from '../../../../utils/constants'; @@ -19,6 +18,7 @@ interface FindCachedUploadPartsOptions { s3Config: ResolvedS3Config; bucket: string; finalKey: string; + resumableUploadsCache: KeyValueStorageInterface; } /** @@ -26,6 +26,7 @@ interface FindCachedUploadPartsOptions { * with ListParts API. If the cached upload is expired(1 hour), return null. */ export const findCachedUploadParts = async ({ + resumableUploadsCache, cacheKey, s3Config, bucket, @@ -35,7 +36,7 @@ export const findCachedUploadParts = async ({ uploadId: string; finalCrc32?: string; } | null> => { - const cachedUploads = await listCachedUploadTasks(defaultStorage); + const cachedUploads = await listCachedUploadTasks(resumableUploadsCache); if ( !cachedUploads[cacheKey] || cachedUploads[cacheKey].lastTouched < Date.now() - ONE_HOUR // Uploads are cached for 1 hour @@ -46,7 +47,7 @@ export const findCachedUploadParts = async ({ const cachedUpload = cachedUploads[cacheKey]; cachedUpload.lastTouched = Date.now(); - await defaultStorage.setItem( + await resumableUploadsCache.setItem( UPLOADS_STORAGE_KEY, JSON.stringify(cachedUploads), ); @@ -65,7 +66,7 @@ export const findCachedUploadParts = async ({ }; } catch (e) { logger.debug('failed to list cached parts, removing cached upload.'); - await removeCachedUpload(cacheKey); + await removeCachedUpload(resumableUploadsCache, cacheKey); return null; } @@ -82,10 +83,12 @@ interface FileMetadata { } const listCachedUploadTasks = async ( - kvStorage: KeyValueStorageInterface, + resumableUploadsCache: KeyValueStorageInterface, ): Promise> => { try { - return JSON.parse((await kvStorage.getItem(UPLOADS_STORAGE_KEY)) ?? '{}'); + return JSON.parse( + (await resumableUploadsCache.getItem(UPLOADS_STORAGE_KEY)) ?? '{}', + ); } catch (e) { logger.debug('failed to parse cached uploads record.'); @@ -136,24 +139,28 @@ export const getUploadsCacheKey = ({ }; export const cacheMultipartUpload = async ( + resumableUploadsCache: KeyValueStorageInterface, cacheKey: string, fileMetadata: Omit, ): Promise => { - const cachedUploads = await listCachedUploadTasks(defaultStorage); + const cachedUploads = await listCachedUploadTasks(resumableUploadsCache); cachedUploads[cacheKey] = { ...fileMetadata, lastTouched: Date.now(), }; - await defaultStorage.setItem( + await resumableUploadsCache.setItem( UPLOADS_STORAGE_KEY, JSON.stringify(cachedUploads), ); }; -export const removeCachedUpload = async (cacheKey: string): Promise => { - const cachedUploads = await listCachedUploadTasks(defaultStorage); +export const removeCachedUpload = async ( + resumableUploadsCache: KeyValueStorageInterface, + cacheKey: string, +): Promise => { + const cachedUploads = await listCachedUploadTasks(resumableUploadsCache); delete cachedUploads[cacheKey]; - await defaultStorage.setItem( + await resumableUploadsCache.setItem( UPLOADS_STORAGE_KEY, JSON.stringify(cachedUploads), ); diff --git a/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/uploadHandlers.ts b/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/uploadHandlers.ts index ece5e4003b9..c23d2cc5052 100644 --- a/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/uploadHandlers.ts +++ b/packages/storage/src/providers/s3/apis/internal/uploadData/multipart/uploadHandlers.ts @@ -1,10 +1,16 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; +import { + Amplify, + KeyValueStorageInterface, + StorageAccessLevel, +} from '@aws-amplify/core'; import { StorageAction } from '@aws-amplify/core/internals/utils'; -import { UploadDataInput, UploadDataWithPathInput } from '../../../../types'; +import { UploadDataInput } from '../../../../types'; +// TODO: Remove this interface when we move to public advanced APIs. +import { UploadDataInput as UploadDataWithPathInputWithAdvancedOptions } from '../../../../../../internals/types/inputs'; import { resolveS3ConfigAndInput, validateStorageOperationInput, @@ -30,6 +36,7 @@ import { import { getStorageUserAgentValue } from '../../../../utils/userAgent'; import { logger } from '../../../../../../utils'; import { validateObjectNotExists } from '../validateObjectNotExists'; +import { StorageOperationOptionsInput } from '../../../../../../types/inputs'; import { uploadPartExecutor } from './uploadPartExecutor'; import { getUploadsCacheKey, removeCachedUpload } from './uploadCache'; @@ -37,6 +44,34 @@ import { getConcurrentUploadsProgressTracker } from './progressTracker'; import { loadOrCreateMultipartUpload } from './initialUpload'; import { getDataChunker } from './getDataChunker'; +type WithResumableCacheConfig> = + Input & { + options?: Input['options'] & { + /** + * The cache instance to store the in-progress multipart uploads so they can be resumed + * after page refresh. By default the library caches the uploaded file name, + * last modified, final checksum, size, bucket, key, and corresponded in-progress + * multipart upload ID from S3. If the library detects the same input corresponds to a + * previously in-progress upload from within 1 hour ago, it will continue + * the upload from where it left. + * + * By default, this option is not set. The upload caching is disabled. + */ + resumableUploadsCache?: KeyValueStorageInterface; + }; + }; + +/** + * The input interface for UploadData API with the options needed for multi-part upload. + * It supports both legacy Gen 1 input with key and Gen2 input with path. It also support additional + * advanced options for StorageBrowser. + * + * @internal + */ +export type MultipartUploadDataInput = WithResumableCacheConfig< + UploadDataInput | UploadDataWithPathInputWithAdvancedOptions +>; + /** * Create closure hiding the multipart upload implementation details and expose the upload job and control functions( * onPause, onResume, onCancel). @@ -44,7 +79,7 @@ import { getDataChunker } from './getDataChunker'; * @internal */ export const getMultipartUploadHandlers = ( - uploadDataInput: UploadDataInput | UploadDataWithPathInput, + uploadDataInput: MultipartUploadDataInput, size?: number, ) => { let resolveCallback: @@ -72,6 +107,8 @@ export const getMultipartUploadHandlers = ( // This should be replaced by a special abort reason. However,the support of this API is lagged behind. let isAbortSignalFromPause = false; + const { resumableUploadsCache } = uploadDataInput.options ?? {}; + const startUpload = async (): Promise => { const { options: uploadDataOptions, data } = uploadDataInput; const resolvedS3Options = await resolveS3ConfigAndInput( @@ -127,6 +164,7 @@ export const getMultipartUploadHandlers = ( data, size, abortSignal: abortController.signal, + resumableUploadsCache, expectedBucketOwner, }); inProgressUpload = { @@ -235,8 +273,8 @@ export const getMultipartUploadHandlers = ( } } - if (uploadCacheKey) { - await removeCachedUpload(uploadCacheKey); + if (resumableUploadsCache && uploadCacheKey) { + await removeCachedUpload(resumableUploadsCache, uploadCacheKey); } const result = { @@ -282,8 +320,8 @@ export const getMultipartUploadHandlers = ( const cancelUpload = async () => { // 2. clear upload cache. - if (uploadCacheKey) { - await removeCachedUpload(uploadCacheKey); + if (uploadCacheKey && resumableUploadsCache) { + await removeCachedUpload(resumableUploadsCache, uploadCacheKey); } // 3. clear multipart upload on server side. await abortMultipartUpload(resolvedS3Config!, { diff --git a/packages/storage/src/providers/s3/apis/internal/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/internal/uploadData/putObjectJob.ts index 08c67f19573..8ebd49f9c4a 100644 --- a/packages/storage/src/providers/s3/apis/internal/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/internal/uploadData/putObjectJob.ts @@ -4,7 +4,9 @@ import { Amplify } from '@aws-amplify/core'; import { StorageAction } from '@aws-amplify/core/internals/utils'; -import { UploadDataInput, UploadDataWithPathInput } from '../../../types'; +import { UploadDataInput } from '../../../types'; +// TODO: Remove this interface when we move to public advanced APIs. +import { UploadDataInput as UploadDataWithPathInputWithAdvancedOptions } from '../../../../../internals/types/inputs'; import { calculateContentMd5, resolveS3ConfigAndInput, @@ -20,6 +22,17 @@ import { constructContentDisposition } from '../../../utils/constructContentDisp import { validateObjectNotExists } from './validateObjectNotExists'; +/** + * The input interface for UploadData API with only the options needed for single part upload. + * It supports both legacy Gen 1 input with key and Gen2 input with path. It also support additional + * advanced options for StorageBrowser. + * + * @internal + */ +export type SinglePartUploadDataInput = + | UploadDataInput + | UploadDataWithPathInputWithAdvancedOptions; + /** * Get a function the returns a promise to call putObject API to S3. * @@ -27,7 +40,7 @@ import { validateObjectNotExists } from './validateObjectNotExists'; */ export const putObjectJob = ( - uploadDataInput: UploadDataInput | UploadDataWithPathInput, + uploadDataInput: SinglePartUploadDataInput, abortSignal: AbortSignal, totalLength?: number, ) => diff --git a/packages/storage/src/providers/s3/apis/uploadData.ts b/packages/storage/src/providers/s3/apis/uploadData.ts index 5ff4fc7a4d1..b6173d3777e 100644 --- a/packages/storage/src/providers/s3/apis/uploadData.ts +++ b/packages/storage/src/providers/s3/apis/uploadData.ts @@ -1,6 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { defaultStorage } from '@aws-amplify/core'; + import { UploadDataInput, UploadDataOutput, @@ -121,5 +123,13 @@ export function uploadData( export function uploadData(input: UploadDataInput): UploadDataOutput; export function uploadData(input: UploadDataInput | UploadDataWithPathInput) { - return uploadDataInternal(input); + return uploadDataInternal({ + ...input, + options: { + ...input?.options, + // This option enables caching in-progress multipart uploads. + // It's ONLY needed for client-side API. + resumableUploadsCache: defaultStorage, + }, + }); }