feat(storage): internal uploadData implementation (#13888)

* feat(storage): internal uploadData implementation

* chore: update bundle size

* fix: address feedbacks

packages/aws-amplify/package.json

@@ -497,7 +497,7 @@
  "name": "[Storage] uploadData (S3)",
  "path": "./dist/esm/storage/index.mjs",
  "import": "{ uploadData }",
- "limit": "22.16 kB"
+ "limit": "22.15 kB"
  }
  ]
 }

packages/storage/__tests__/internals/apis/uploadData.test.ts

@@ -0,0 +1,67 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { uploadData as advancedUploadData } from '../../../src/internals';
+import { uploadData as uploadDataInternal } from '../../../src/providers/s3/apis/internal/uploadData';
+
+jest.mock('../../../src/providers/s3/apis/internal/uploadData');
+const mockedUploadDataInternal = jest.mocked(uploadDataInternal);
+const mockedUploadTask = 'UPLOAD_TASK';
+
+describe('uploadData (internal)', () => {
+ beforeEach(() => {
+ mockedUploadDataInternal.mockReturnValue(mockedUploadTask as any);
+ });
+
+ afterEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it('should pass advanced option locationCredentialsProvider to internal remove', async () => {
+ const useAccelerateEndpoint = true;
+ const bucket = { bucketName: 'bucket', region: 'us-east-1' };
+ const locationCredentialsProvider = async () => ({
+ credentials: {
+ accessKeyId: 'akid',
+ secretAccessKey: 'secret',
+ sessionToken: 'token',
+ expiration: new Date(),
+ },
+ });
+ const contentDisposition = { type: 'attachment', filename: 'foo' } as const;
+ const onProgress = jest.fn();
+ const metadata = { foo: 'bar' };
+
+ const result = advancedUploadData({
+ path: 'input/path/to/mock/object',
+ data: 'data',
+ options: {
+ useAccelerateEndpoint,
+ bucket,
+ locationCredentialsProvider,
+ contentDisposition,
+ contentEncoding: 'gzip',
+ contentType: 'text/html',
+ onProgress,
+ metadata,
+ },
+ });
+
+ expect(mockedUploadDataInternal).toHaveBeenCalledTimes(1);
+ expect(mockedUploadDataInternal).toHaveBeenCalledWith({
+ path: 'input/path/to/mock/object',
+ data: 'data',
+ options: {
+ useAccelerateEndpoint,
+ bucket,
+ locationCredentialsProvider,
+ contentDisposition,
+ contentEncoding: 'gzip',
+ contentType: 'text/html',
+ onProgress,
+ metadata,
+ },
+ });
+ expect(result).toEqual(mockedUploadTask);
+ });
+});

packages/storage/__tests__/providers/s3/apis/uploadData/byteLength.test.ts

@@ -1,7 +1,7 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { byteLength } from '../../../../../src/providers/s3/apis/uploadData/byteLength';
+import { byteLength } from '../../../../../src/providers/s3/apis/internal/uploadData/byteLength';
 
 describe('byteLength', () => {
  it('returns 0 for null or undefined', () => {

packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts

@@ -8,13 +8,15 @@ import {
  StorageValidationErrorCode,
  validationErrorMap,
 } from '../../../../../src/errors/types/validation';
-import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob';
-import { getMultipartUploadHandlers } from '../../../../../src/providers/s3/apis/uploadData/multipart';
+import { putObjectJob } from '../../../../../src/providers/s3/apis/internal/uploadData/putObjectJob';
+import { getMultipartUploadHandlers } from '../../../../../src/providers/s3/apis/internal/uploadData/multipart';
 import { UploadDataInput, UploadDataWithPathInput } from '../../../../../src';
 
 jest.mock('../../../../../src/providers/s3/utils/');
-jest.mock('../../../../../src/providers/s3/apis/uploadData/putObjectJob');
-jest.mock('../../../../../src/providers/s3/apis/uploadData/multipart');
+jest.mock(
+ '../../../../../src/providers/s3/apis/internal/uploadData/putObjectJob',
+);
+jest.mock('../../../../../src/providers/s3/apis/internal/uploadData/multipart');
 
 const testPath = 'testPath/object';
 const mockCreateUploadTask = createUploadTask as jest.Mock;

packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts

@@ -15,7 +15,7 @@ import {
  listParts,
  uploadPart,
 } from '../../../../../src/providers/s3/utils/client/s3data';
-import { getMultipartUploadHandlers } from '../../../../../src/providers/s3/apis/uploadData/multipart';
+import { getMultipartUploadHandlers } from '../../../../../src/providers/s3/apis/internal/uploadData/multipart';
 import {
  StorageValidationErrorCode,
  validationErrorMap,
@@ -24,7 +24,7 @@ import {
  CHECKSUM_ALGORITHM_CRC32,
  UPLOADS_STORAGE_KEY,
 } from '../../../../../src/providers/s3/utils/constants';
-import { byteLength } from '../../../../../src/providers/s3/apis/uploadData/byteLength';
+import { byteLength } from '../../../../../src/providers/s3/apis/internal/uploadData/byteLength';
 import { CanceledError } from '../../../../../src/errors/CanceledError';
 import { StorageOptions } from '../../../../../src/types';
 import '../testUtils';

packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts

@@ -13,7 +13,7 @@ import {
 } from '../../../../../src/providers/s3/utils/client/s3data';
 import { calculateContentMd5 } from '../../../../../src/providers/s3/utils';
 import * as CRC32 from '../../../../../src/providers/s3/utils/crc32';
-import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob';
+import { putObjectJob } from '../../../../../src/providers/s3/apis/internal/uploadData/putObjectJob';
 import '../testUtils';
 import { UploadDataChecksumAlgorithm } from '../../../../../src/providers/s3/types/options';
 import { CHECKSUM_ALGORITHM_CRC32 } from '../../../../../src/providers/s3/utils/constants';

packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.native.test.ts

@@ -9,7 +9,7 @@ import {
 } from 'node:util';
 
 import { getCombinedCrc32 } from '../../../../src/providers/s3/utils/getCombinedCrc32.native';
-import { byteLength } from '../../../../src/providers/s3/apis/uploadData/byteLength';
+import { byteLength } from '../../../../src/providers/s3/apis/internal/uploadData/byteLength';
 
 global.Blob = BlobPolyfill as any;
 global.File = FilePolyfill as any;

packages/storage/__tests__/providers/s3/utils/getCombinedCrc32.test.ts

@@ -9,7 +9,7 @@ import {
 } from 'node:util';
 
 import { getCombinedCrc32 } from '../../../../src/providers/s3/utils/getCombinedCrc32';
-import { byteLength } from '../../../../src/providers/s3/apis/uploadData/byteLength';
+import { byteLength } from '../../../../src/providers/s3/apis/internal/uploadData/byteLength';
 
 global.Blob = BlobPolyfill as any;
 global.File = FilePolyfill as any;

packages/storage/src/internals/apis/getProperties.ts

@@ -17,10 +17,10 @@ import { GetPropertiesOutput } from '../types/outputs';
  *
  * @internal
  */
-export function getProperties(
+export const getProperties = (
  input: GetPropertiesInput,
-): Promise<GetPropertiesOutput> {
- return getPropertiesInternal(Amplify, {
+): Promise<GetPropertiesOutput> =>
+ getPropertiesInternal(Amplify, {
  path: input.path,
  options: {
  useAccelerateEndpoint: input?.options?.useAccelerateEndpoint,
@@ -30,4 +30,3 @@ export function getProperties(
  // Type casting is necessary because `getPropertiesInternal` supports both Gen1 and Gen2 signatures, but here
  // given in input can only be Gen2 signature, the return can only ben Gen2 signature.
  }) as Promise<GetPropertiesOutput>;
-}

packages/storage/src/internals/apis/getUrl.ts

@@ -10,8 +10,8 @@ import { GetUrlOutput } from '../types/outputs';
 /**
  * @internal
  */
-export function getUrl(input: GetUrlInput) {
- return getUrlInternal(Amplify, {
+export const getUrl = (input: GetUrlInput) =>
+ getUrlInternal(Amplify, {
  path: input.path,
  options: {
  useAccelerateEndpoint: input?.options?.useAccelerateEndpoint,
@@ -27,4 +27,3 @@ export function getUrl(input: GetUrlInput) {
  // Type casting is necessary because `getPropertiesInternal` supports both Gen1 and Gen2 signatures, but here
  // given in input can only be Gen2 signature, the return can only ben Gen2 signature.
  }) as Promise<GetUrlOutput>;
-}

packages/storage/src/internals/apis/uploadData.ts

@@ -0,0 +1,33 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { UploadDataInput } from '../types/inputs';
+import { UploadDataOutput } from '../types/outputs';
+import { uploadData as uploadDataInternal } from '../../providers/s3/apis/internal/uploadData';
+
+/**
+ * @internal
+ */
+export const uploadData = (input: UploadDataInput) => {
+ const { data, path, options } = input;
+
+ return uploadDataInternal({
+ path,
+ data,
+ options: {
+ useAccelerateEndpoint: options?.useAccelerateEndpoint,
+ bucket: options?.bucket,
+ onProgress: options?.onProgress,
+ contentDisposition: options?.contentDisposition,
+ contentEncoding: options?.contentEncoding,
+ contentType: options?.contentType,
+ metadata: options?.metadata,
+ preventOverwrite: options?.preventOverwrite,
+
+ // Advanced options
+ locationCredentialsProvider: options?.locationCredentialsProvider,
+ },
+ // Type casting is necessary because `uploadDataInternal` supports both Gen1 and Gen2 signatures, but here
+ // given in input can only be Gen2 signature, the return can only ben Gen2 signature.
+ }) as UploadDataOutput;
+};

packages/storage/src/internals/index.ts

@@ -16,6 +16,7 @@ export {
  CopyInput,
  ListInput,
  RemoveInput,
+ UploadDataInput,
  DownloadDataInput,
 } from './types/inputs';
 export {
@@ -24,6 +25,7 @@ export {
  GetPropertiesOutput,
  GetUrlOutput,
  RemoveOutput,
+ UploadDataOutput,
  DownloadDataOutput,
  ListOutput,
  CopyOutput,
@@ -35,6 +37,7 @@ export { list } from './apis/list';
 export { getProperties } from './apis/getProperties';
 export { getUrl } from './apis/getUrl';
 export { remove } from './apis/remove';
+export { uploadData } from './apis/uploadData';
 export { downloadData } from './apis/downloadData';
 export { copy } from './apis/copy';
 

packages/storage/src/internals/types/inputs.ts

@@ -12,6 +12,7 @@ import {
  GetPropertiesWithPathInput,
  GetUrlWithPathInput,
  RemoveWithPathInput,
+ UploadDataWithPathInput,
 } from '../../providers/s3';
 import {
  ListAllWithPathInput,
@@ -94,6 +95,13 @@ export type CopyInput = ExtendCopyInputWithAdvancedOptions<
  }
 >;
 
+export type UploadDataInput = ExtendInputWithAdvancedOptions<
+ UploadDataWithPathInput,
+ {
+ locationCredentialsProvider?: CredentialsProvider;
+ }
+>;
+
 /**
  * @internal
  */
@@ -111,8 +119,7 @@ export type DownloadDataInput = ExtendInputWithAdvancedOptions<
 type ExtendInputWithAdvancedOptions<InputType, ExtendedOptionsType> =
  InputType extends StorageOperationInputWithPath &
  StorageOperationOptionsInput<infer PublicInputOptionsType>
- ? {
- path: InputType['path'];
+ ? InputType & {
  options?: PublicInputOptionsType & ExtendedOptionsType;
  }
  : never;

packages/storage/src/internals/types/outputs.ts

@@ -9,6 +9,7 @@ import {
  ListAllWithPathOutput,
  ListPaginateWithPathOutput,
  RemoveWithPathOutput,
+ UploadDataWithPathOutput,
 } from '../../providers/s3/types';
 
 import { ListLocationsOutput, LocationCredentials } from './credentials';
@@ -48,6 +49,11 @@ export type RemoveOutput = RemoveWithPathOutput;
  */
 export type ListOutput = ListAllWithPathOutput | ListPaginateWithPathOutput;
 
+/**
+ * @internal
+ */
+export type UploadDataOutput = UploadDataWithPathOutput;
+
 /**
  * @internal
  */

packages/storage/src/providers/s3/apis/internal/uploadData/index.ts

@@ -0,0 +1,53 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { UploadDataInput } from '../../../types';
+// TODO: Remove this interface when we move to public advanced APIs.
+import { UploadDataInput as UploadDataWithPathInputWithAdvancedOptions } from '../../../../../internals/types/inputs';
+import { createUploadTask } from '../../../utils';
+import { assertValidationError } from '../../../../../errors/utils/assertValidationError';
+import { StorageValidationErrorCode } from '../../../../../errors/types/validation';
+import { DEFAULT_PART_SIZE, MAX_OBJECT_SIZE } from '../../../utils/constants';
+
+import { byteLength } from './byteLength';
+import { putObjectJob } from './putObjectJob';
+import { getMultipartUploadHandlers } from './multipart';
+
+export const uploadData = (
+ input: UploadDataInput | UploadDataWithPathInputWithAdvancedOptions,
+) => {
+ const { data } = input;
+
+ const dataByteLength = byteLength(data);
+ assertValidationError(
+ dataByteLength === undefined || dataByteLength <= MAX_OBJECT_SIZE,
+ StorageValidationErrorCode.ObjectIsTooLarge,
+ );
+
+ if (dataByteLength && dataByteLength <= DEFAULT_PART_SIZE) {
+ // Single part upload
+ const abortController = new AbortController();
+
+ return createUploadTask({
+ isMultipartUpload: false,
+ job: putObjectJob(input, abortController.signal, dataByteLength),
+ onCancel: (message?: string) => {
+ abortController.abort(message);
+ },
+ });
+ } else {
+ // Multipart upload
+ const { multipartUploadJob, onPause, onResume, onCancel } =
+ getMultipartUploadHandlers(input, dataByteLength);
+
+ return createUploadTask({
+ isMultipartUpload: true,
+ job: multipartUploadJob,
+ onCancel: (message?: string) => {
+ onCancel(message);
+ },
+ onPause,
+ onResume,
+ });
+ }
+};

packages/storage/src/providers/s3/apis/uploadData/multipart/calculatePartSize.ts → packages/storage/src/providers/s3/apis/internal/uploadData/multipart/calculatePartSize.ts

@@ -1,7 +1,10 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { DEFAULT_PART_SIZE, MAX_PARTS_COUNT } from '../../../utils/constants';
+import {
+ DEFAULT_PART_SIZE,
+ MAX_PARTS_COUNT,
+} from '../../../../utils/constants';
 
 export const calculatePartSize = (totalSize?: number): number => {
  if (!totalSize) {

packages/storage/src/providers/s3/apis/uploadData/multipart/getDataChunker.ts → packages/storage/src/providers/s3/apis/internal/uploadData/multipart/getDataChunker.ts

@@ -1,12 +1,12 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { StorageUploadDataPayload } from '../../../../../types';
+import { StorageUploadDataPayload } from '../../../../../../types';
 import {
  StorageValidationErrorCode,
  validationErrorMap,
-} from '../../../../../errors/types/validation';
-import { StorageError } from '../../../../../errors/StorageError';
+} from '../../../../../../errors/types/validation';
+import { StorageError } from '../../../../../../errors/StorageError';
 
 import { calculatePartSize } from './calculatePartSize';
 

packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts → packages/storage/src/providers/s3/apis/internal/uploadData/multipart/initialUpload.ts

@@ -7,13 +7,13 @@ import {
  ContentDisposition,
  ResolvedS3Config,
  UploadDataChecksumAlgorithm,
-} from '../../../types/options';
-import { StorageUploadDataPayload } from '../../../../../types';
-import { Part, createMultipartUpload } from '../../../utils/client/s3data';
-import { logger } from '../../../../../utils';
-import { constructContentDisposition } from '../../../utils/constructContentDisposition';
-import { getCombinedCrc32 } from '../../../utils/getCombinedCrc32';
-import { CHECKSUM_ALGORITHM_CRC32 } from '../../../utils/constants';
+} from '../../../../types/options';
+import { StorageUploadDataPayload } from '../../../../../../types';
+import { Part, createMultipartUpload } from '../../../../utils/client/s3data';
+import { logger } from '../../../../../../utils';
+import { constructContentDisposition } from '../../../../utils/constructContentDisposition';
+import { getCombinedCrc32 } from '../../../../utils/getCombinedCrc32';
+import { CHECKSUM_ALGORITHM_CRC32 } from '../../../../utils/constants';
 
 import {
  cacheMultipartUpload,