diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index a3c44a89861..6104db0a485 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -485,7 +485,7 @@ "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "14.94 kB" + "limit": "15.04 kB" }, { "name": "[Storage] remove (S3)", diff --git a/packages/storage/__tests__/providers/s3/apis/list.test.ts b/packages/storage/__tests__/providers/s3/apis/list.test.ts index 9629129d7a2..348719732c0 100644 --- a/packages/storage/__tests__/providers/s3/apis/list.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/list.test.ts @@ -512,4 +512,116 @@ describe('list API', () => { } }); }); + + describe('with delimiter', () => { + const mockedContents = [ + { + Key: 'photos/', + ...listObjectClientBaseResultItem, + }, + { + Key: 'photos/2023.png', + ...listObjectClientBaseResultItem, + }, + { + Key: 'photos/2024.png', + ...listObjectClientBaseResultItem, + }, + ]; + const mockedCommonPrefixes = [ + { Prefix: 'photos/2023/' }, + { Prefix: 'photos/2024/' }, + { Prefix: 'photos/2025/' }, + ]; + + const mockedPath = 'photos/'; + + beforeEach(() => { + mockListObject.mockResolvedValueOnce({ + Contents: mockedContents, + CommonPrefixes: mockedCommonPrefixes, + }); + }); + afterEach(() => { + jest.clearAllMocks(); + mockListObject.mockClear(); + }); + + it('should return subpaths when delimiter is passed in the request', async () => { + const { items, subpaths } = await list({ + path: mockedPath, + options: { + delimiter: '/', + }, + }); + expect(items).toHaveLength(3); + expect(subpaths).toEqual([ + 'photos/2023/', + 'photos/2024/', + 'photos/2025/', + ]); + expect(listObjectsV2).toHaveBeenCalledTimes(1); + await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( + listObjectClientConfig, + { + Bucket: bucket, + MaxKeys: 1000, + Prefix: mockedPath, + Delimiter: '/', + }, + ); + }); + + it('should return subpaths when delimiter and listAll are passed in the request', async () => { + const { items, subpaths } = await list({ + path: mockedPath, + options: { + delimiter: '/', + listAll: true, + }, + }); + expect(items).toHaveLength(3); + expect(subpaths).toEqual([ + 'photos/2023/', + 'photos/2024/', + 'photos/2025/', + ]); + expect(listObjectsV2).toHaveBeenCalledTimes(1); + await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( + listObjectClientConfig, + { + Bucket: bucket, + MaxKeys: 1000, + Prefix: mockedPath, + Delimiter: '/', + }, + ); + }); + + it('should return subpaths when delimiter is pageSize are passed in the request', async () => { + const { items, subpaths } = await list({ + path: mockedPath, + options: { + delimiter: '/', + pageSize: 3, + }, + }); + expect(items).toHaveLength(3); + expect(subpaths).toEqual([ + 'photos/2023/', + 'photos/2024/', + 'photos/2025/', + ]); + expect(listObjectsV2).toHaveBeenCalledTimes(1); + await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( + listObjectClientConfig, + { + Bucket: bucket, + MaxKeys: 3, + Prefix: mockedPath, + Delimiter: '/', + }, + ); + }); + }); }); diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts index f180dfe5247..7b625263a84 100644 --- a/packages/storage/src/providers/s3/apis/internal/list.ts +++ b/packages/storage/src/providers/s3/apis/internal/list.ts @@ -29,6 +29,7 @@ import { import { getStorageUserAgentValue } from '../../utils/userAgent'; import { logger } from '../../../../utils'; import { STORAGE_INPUT_PREFIX } from '../../utils/constants'; +import { CommonPrefix } from '../../utils/client/types'; const MAX_PAGE_SIZE = 1000; @@ -79,6 +80,7 @@ export const list = async ( Prefix: isInputWithPrefix ? `${generatedPrefix}${objectKey}` : objectKey, MaxKeys: options?.listAll ? undefined : options?.pageSize, ContinuationToken: options?.listAll ? undefined : options?.nextToken, + Delimiter: options?.delimiter, }; logger.debug(`listing items from "${listParams.Prefix}"`); @@ -176,23 +178,29 @@ const _listAllWithPath = async ({ listParams, }: ListInputArgs): Promise => { const listResult: ListOutputItemWithPath[] = []; + const subpaths: string[] = []; let continuationToken = listParams.ContinuationToken; do { - const { items: pageResults, nextToken: pageNextToken } = - await _listWithPath({ - s3Config, - listParams: { - ...listParams, - ContinuationToken: continuationToken, - MaxKeys: MAX_PAGE_SIZE, - }, - }); + const { + items: pageResults, + subpaths: pageSubpaths, + nextToken: pageNextToken, + } = await _listWithPath({ + s3Config, + listParams: { + ...listParams, + ContinuationToken: continuationToken, + MaxKeys: MAX_PAGE_SIZE, + }, + }); listResult.push(...pageResults); + subpaths.push(...(pageSubpaths ?? [])); continuationToken = pageNextToken; } while (continuationToken); return { items: listResult, + ...parseSubpaths(subpaths), }; }; @@ -206,7 +214,11 @@ const _listWithPath = async ({ listParamsClone.MaxKeys = MAX_PAGE_SIZE; } - const response: ListObjectsV2Output = await listObjectsV2( + const { + Contents: contents, + NextContinuationToken: nextContinuationToken, + CommonPrefixes: commonPrefixes, + }: ListObjectsV2Output = await listObjectsV2( { ...s3Config, userAgentValue: getStorageUserAgentValue(StorageAction.List), @@ -214,19 +226,35 @@ const _listWithPath = async ({ listParamsClone, ); - if (!response?.Contents) { + const subpaths = mapCommonPrefixesToSubpaths(commonPrefixes); + + if (!contents) { return { items: [], + ...parseSubpaths(subpaths), }; } return { - items: response.Contents.map(item => ({ + items: contents.map(item => ({ path: item.Key!, eTag: item.ETag, lastModified: item.LastModified, size: item.Size, })), - nextToken: response.NextContinuationToken, + nextToken: nextContinuationToken, + ...parseSubpaths(subpaths), }; }; + +function mapCommonPrefixesToSubpaths( + commonPrefixes?: CommonPrefix[], +): string[] | undefined { + const mappedSubpaths = commonPrefixes?.map(({ Prefix }) => Prefix); + + return mappedSubpaths?.filter((subpath): subpath is string => !!subpath); +} + +function parseSubpaths(subpaths?: string[]) { + return subpaths && subpaths.length > 0 ? { subpaths } : {}; +} diff --git a/packages/storage/src/types/options.ts b/packages/storage/src/types/options.ts index b9c74590ba6..31e371593f5 100644 --- a/packages/storage/src/types/options.ts +++ b/packages/storage/src/types/options.ts @@ -10,12 +10,14 @@ export interface StorageOptions { export type StorageListAllOptions = StorageOptions & { listAll: true; + delimiter?: string; }; export type StorageListPaginateOptions = StorageOptions & { listAll?: false; pageSize?: number; nextToken?: string; + delimiter?: string; }; export type StorageRemoveOptions = StorageOptions; diff --git a/packages/storage/src/types/outputs.ts b/packages/storage/src/types/outputs.ts index e38482729b8..312966b087b 100644 --- a/packages/storage/src/types/outputs.ts +++ b/packages/storage/src/types/outputs.ts @@ -70,4 +70,9 @@ export interface StorageListOutput { * List of items returned by the list API. */ items: Item[]; + /** + * List of subpaths returned by the list API when a delimiter option is passed + * in the request of the list API. + */ + subpaths?: string[]; }