Skip to content

Commit

Permalink
feat(storage): add advanced option to disable upload cache (#13931)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AllanZhengYP
AllanZhengYP authored Oct 17, 2024
1 parent 2a9053f commit 5145064
Show file tree
Hide file tree
Showing 11 changed files with 231 additions and 54 deletions.
2 changes: 1 addition & 1 deletion packages/aws-amplify/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,7 @@
"name": "[Storage] uploadData (S3)",
"path": "./dist/esm/storage/index.mjs",
"import": "{ uploadData }",
"limit": "22.37 kB"
"limit": "22.39 kB"
}
]
}
34 changes: 29 additions & 5 deletions packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

import { defaultStorage } from '@aws-amplify/core';

import { uploadData } from '../../../../../src/providers/s3/apis';
import { MAX_OBJECT_SIZE } from '../../../../../src/providers/s3/utils/constants';
import { createUploadTask } from '../../../../../src/providers/s3/utils';
Expand Down Expand Up @@ -70,6 +72,22 @@ describe('uploadData with key', () => {
expect(mockGetMultipartUploadHandlers).not.toHaveBeenCalled();
});

it('should use putObject for 0 bytes data (e.g. create a folder)', () => {
const testInput = {
key: 'key',
data: '', // 0 bytes
};

uploadData(testInput);

expect(mockPutObjectJob).toHaveBeenCalledWith(
expect.objectContaining(testInput),
expect.any(AbortSignal),
expect.any(Number),
);
expect(mockGetMultipartUploadHandlers).not.toHaveBeenCalled();
});

it('should use uploadTask', async () => {
mockPutObjectJob.mockReturnValueOnce('putObjectJob');
mockCreateUploadTask.mockReturnValueOnce('uploadTask');
Expand Down Expand Up @@ -175,7 +193,7 @@ describe('uploadData with path', () => {
uploadData(testInput);

expect(mockPutObjectJob).toHaveBeenCalledWith(
testInput,
expect.objectContaining(testInput),
expect.any(AbortSignal),
expect.any(Number),
);
Expand All @@ -192,7 +210,7 @@ describe('uploadData with path', () => {
uploadData(testInput);

expect(mockPutObjectJob).toHaveBeenCalledWith(
testInput,
expect.objectContaining(testInput),
expect.any(AbortSignal),
expect.any(Number),
);
Expand Down Expand Up @@ -231,7 +249,12 @@ describe('uploadData with path', () => {

expect(mockPutObjectJob).not.toHaveBeenCalled();
expect(mockGetMultipartUploadHandlers).toHaveBeenCalledWith(
testInput,
expect.objectContaining({
...testInput,
options: {
resumableUploadsCache: defaultStorage,
},
}),
expect.any(Number),
);
});
Expand Down Expand Up @@ -291,9 +314,10 @@ describe('uploadData with path', () => {
};
uploadData(testInput);
expect(mockGetMultipartUploadHandlers).toHaveBeenCalledWith(
expect.objectContaining({
{
...testInput,
}),
options: expect.objectContaining(testInput.options),
},
expect.any(Number),
);

Expand Down
93 changes: 83 additions & 10 deletions packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -524,13 +524,33 @@ describe('getMultipartUploadHandlers with key', () => {
mockDefaultStorage.setItem.mockReset();
});

it('should disable upload caching if resumableUploadsCache option is not set', async () => {
mockMultipartUploadSuccess();
const size = 8 * MB;
const { multipartUploadJob } = getMultipartUploadHandlers(
{
key: defaultKey,
data: new ArrayBuffer(size),
},
size,
);
await multipartUploadJob();
expect(mockDefaultStorage.getItem).not.toHaveBeenCalled();
expect(mockDefaultStorage.setItem).not.toHaveBeenCalled();
expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1);
expect(mockListParts).not.toHaveBeenCalled();
});

it('should send createMultipartUpload request if the upload task is not cached', async () => {
mockMultipartUploadSuccess();
const size = 8 * MB;
const { multipartUploadJob } = getMultipartUploadHandlers(
{
key: defaultKey,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand Down Expand Up @@ -559,6 +579,9 @@ describe('getMultipartUploadHandlers with key', () => {
{
key: defaultKey,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand All @@ -577,6 +600,9 @@ describe('getMultipartUploadHandlers with key', () => {
{
key: defaultKey,
data: new File([new ArrayBuffer(size)], 'someName'),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand Down Expand Up @@ -612,6 +638,9 @@ describe('getMultipartUploadHandlers with key', () => {
{
key: defaultKey,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand All @@ -630,6 +659,9 @@ describe('getMultipartUploadHandlers with key', () => {
{
key: defaultKey,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand Down Expand Up @@ -657,6 +689,9 @@ describe('getMultipartUploadHandlers with key', () => {
{
key: defaultKey,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand All @@ -679,6 +714,9 @@ describe('getMultipartUploadHandlers with key', () => {
{
key: defaultKey,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand Down Expand Up @@ -795,9 +833,7 @@ describe('getMultipartUploadHandlers with key', () => {
it('should send progress for cached upload parts', async () => {
mockMultipartUploadSuccess();

const mockDefaultStorage = defaultStorage as jest.Mocked<
typeof defaultStorage
>;
const mockDefaultStorage = jest.mocked(defaultStorage);
mockDefaultStorage.getItem.mockResolvedValue(
JSON.stringify({
[defaultCacheKey]: {
Expand All @@ -819,6 +855,7 @@ describe('getMultipartUploadHandlers with key', () => {
data: new ArrayBuffer(8 * MB),
options: {
onProgress,
resumableUploadsCache: mockDefaultStorage,
},
},
8 * MB,
Expand Down Expand Up @@ -1258,13 +1295,33 @@ describe('getMultipartUploadHandlers with path', () => {
mockDefaultStorage.setItem.mockReset();
});

it('should disable upload caching if resumableUploadsCache option is not set', async () => {
mockMultipartUploadSuccess();
const size = 8 * MB;
const { multipartUploadJob } = getMultipartUploadHandlers(
{
key: defaultKey,
data: new ArrayBuffer(size),
},
size,
);
await multipartUploadJob();
expect(mockDefaultStorage.getItem).not.toHaveBeenCalled();
expect(mockDefaultStorage.setItem).not.toHaveBeenCalled();
expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1);
expect(mockListParts).not.toHaveBeenCalled();
});

it('should send createMultipartUpload request if the upload task is not cached', async () => {
mockMultipartUploadSuccess();
const size = 8 * MB;
const { multipartUploadJob } = getMultipartUploadHandlers(
{
path: testPath,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand Down Expand Up @@ -1293,6 +1350,9 @@ describe('getMultipartUploadHandlers with path', () => {
{
path: testPath,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand All @@ -1311,6 +1371,9 @@ describe('getMultipartUploadHandlers with path', () => {
{
path: testPath,
data: new File([new ArrayBuffer(size)], 'someName'),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand All @@ -1321,12 +1384,10 @@ describe('getMultipartUploadHandlers with path', () => {
mockDefaultStorage.setItem.mock.calls[0][1],
);

// \d{13} is the file lastModified property of a file
const lastModifiedRegex = /someName_\d{13}_/;

expect(Object.keys(cacheValue)).toEqual([
expect.stringMatching(
new RegExp(lastModifiedRegex.source + testPathCacheKey),
// \d{13} is the file lastModified property of a file
new RegExp('someName_\\d{13}_' + testPathCacheKey),
),
]);
});
Expand All @@ -1349,6 +1410,9 @@ describe('getMultipartUploadHandlers with path', () => {
{
path: testPath,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand All @@ -1367,6 +1431,9 @@ describe('getMultipartUploadHandlers with path', () => {
{
path: testPath,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand All @@ -1392,6 +1459,9 @@ describe('getMultipartUploadHandlers with path', () => {
{
path: testPath,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand All @@ -1414,6 +1484,9 @@ describe('getMultipartUploadHandlers with path', () => {
{
path: testPath,
data: new ArrayBuffer(size),
options: {
resumableUploadsCache: mockDefaultStorage,
},
},
size,
);
Expand Down Expand Up @@ -1530,9 +1603,8 @@ describe('getMultipartUploadHandlers with path', () => {
it('should send progress for cached upload parts', async () => {
mockMultipartUploadSuccess();

const mockDefaultStorage = defaultStorage as jest.Mocked<
typeof defaultStorage
>;
const mockDefaultStorage = jest.mocked(defaultStorage);

mockDefaultStorage.getItem.mockResolvedValue(
JSON.stringify({
[testPathCacheKey]: {
Expand All @@ -1554,6 +1626,7 @@ describe('getMultipartUploadHandlers with path', () => {
data: new ArrayBuffer(8 * MB),
options: {
onProgress,
resumableUploadsCache: mockDefaultStorage,
},
},
8 * MB,
Expand Down
6 changes: 2 additions & 4 deletions packages/storage/src/internals/types/inputs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,11 @@ import {
DownloadDataWithPathInput,
GetPropertiesWithPathInput,
GetUrlWithPathInput,
ListAllWithPathInput,
ListPaginateWithPathInput,
RemoveWithPathInput,
UploadDataWithPathInput,
} from '../../providers/s3';
import {
ListAllWithPathInput,
ListPaginateWithPathInput,
} from '../../providers/s3/types/inputs';

import { CredentialsProvider, ListLocationsInput } from './credentials';
import { Permission, PrefixType, Privilege } from './common';
Expand Down
12 changes: 6 additions & 6 deletions packages/storage/src/providers/s3/apis/internal/uploadData/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

import { UploadDataInput } from '../../../types';
// TODO: Remove this interface when we move to public advanced APIs.
import { UploadDataInput as UploadDataWithPathInputWithAdvancedOptions } from '../../../../../internals/types/inputs';
import { createUploadTask } from '../../../utils';
import { assertValidationError } from '../../../../../errors/utils/assertValidationError';
import { StorageValidationErrorCode } from '../../../../../errors/types/validation';
import { DEFAULT_PART_SIZE, MAX_OBJECT_SIZE } from '../../../utils/constants';

import { byteLength } from './byteLength';
import { putObjectJob } from './putObjectJob';
import { getMultipartUploadHandlers } from './multipart';
import { SinglePartUploadDataInput, putObjectJob } from './putObjectJob';
import {
MultipartUploadDataInput,
getMultipartUploadHandlers,
} from './multipart';

export const uploadData = (
input: UploadDataInput | UploadDataWithPathInputWithAdvancedOptions,
input: SinglePartUploadDataInput | MultipartUploadDataInput,
) => {
const { data } = input;

Expand Down
5 changes: 4 additions & 1 deletion packages/storage/src/providers/s3/apis/internal/uploadData/multipart/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

export { getMultipartUploadHandlers } from './uploadHandlers';
export {
getMultipartUploadHandlers,
MultipartUploadDataInput,
} from './uploadHandlers';
Loading

0 comments on commit 5145064

Please sign in to comment.