Skip to content

Commit

Permalink
feat(storage): multi bucket list api (#5576)
Browse files Browse the repository at this point in the history 
* added bucket option to list api options and added tests

* updated tests to account for changes to uploadData api

---------

Co-authored-by: ekjotmultani <ekjotm@amazon.com>
Co-authored-by: NikaHsn <nika.hasani@gmail.com>
Co-authored-by: Tyler-Larkin <tyllark@amazon.com>
Co-authored-by: Elijah Quartey <Equartey@users.noreply.github.com>
  • Loading branch information
@ekjotmultani @NikaHsn
@tyllark @Equartey
5 people authored and Nika Hassani committed Dec 3, 2024
1 parent fc1293c commit 59ebae6
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 15 deletions.
9 changes: 7 additions & 2 deletions packages/amplify_core/lib/src/types/storage/list_options.dart
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

import 'package:aws_common/aws_common.dart';
import 'package:amplify_core/amplify_core.dart';

/// {@template amplify_core.storage.list_options}
/// Configurable options for `Amplify.Storage.list`.
Expand All @@ -15,6 +15,7 @@ class StorageListOptions
const StorageListOptions({
this.pageSize = 1000,
this.nextToken,
this.bucket,
this.pluginOptions,
});

Expand All @@ -27,8 +28,11 @@ class StorageListOptions
/// {@macro amplify_core.storage.list_plugin_options}
final StorageListPluginOptions? pluginOptions;

/// Optionally specify which bucket to retrieve
final StorageBucket? bucket;

@override
List<Object?> get props => [pageSize, nextToken, pluginOptions];
List<Object?> get props => [pageSize, nextToken, pluginOptions, bucket];

@override
String get runtimeTypeName => 'StorageListOptions';
Expand All @@ -37,6 +41,7 @@ class StorageListOptions
Map<String, Object?> toJson() => {
'pageSize': pageSize,
'nextToken': nextToken,
'bucket': bucket,
'pluginOptions': pluginOptions?.toJson(),
};
}
Expand Down
111 changes: 102 additions & 9 deletions packages/storage/amplify_storage_s3/example/integration_test/list_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,32 +20,73 @@ void main() {
'$uniquePrefix/file2.txt',
'$uniquePrefix/subdir/file3.txt',
'$uniquePrefix/subdir2#file4.txt',
'$uniquePrefix/file5.txt',
'$uniquePrefix/file6.txt',
'$uniquePrefix/subdir3/file7.txt',
'$uniquePrefix/subdir4#file8.txt',
];
group('standard config', () {
final mainBucket =
StorageBucket.fromOutputs('Storage Integ Test main bucket');
final secondaryBucket = StorageBucket.fromOutputs(
'Storage Integ Test secondary bucket',
);
setUpAll(() async {
await configure(amplifyEnvironments['main']!);

for (final path in uploadedPaths) {
for (var pathIndex = 0;
pathIndex < uploadedPaths.length ~/ 2;
pathIndex++) {
await Amplify.Storage.uploadData(
path: StoragePath.fromString(path),
path: StoragePath.fromString(uploadedPaths[pathIndex]),
data: StorageDataPayload.bytes('test content'.codeUnits),
options: StorageUploadDataOptions(
bucket: mainBucket,
),
).result;
}
for (var pathIndex = uploadedPaths.length ~/ 2;
pathIndex < uploadedPaths.length;
pathIndex++) {
await Amplify.Storage.uploadData(
path: StoragePath.fromString(uploadedPaths[pathIndex]),
data: StorageDataPayload.bytes('test content'.codeUnits),
options: StorageUploadDataOptions(
bucket: secondaryBucket,
),
).result;
}

for (final path in uploadedPaths) {
addTearDownPath(StoragePath.fromString(path));
}
});

group('list() without options', () {
testWidgets('should list all files with unique prefix', (_) async {
final listResult = await Amplify.Storage.list(
// this will use the main bucket by default when no optional bucket is specified
final listResultMainBucket = await Amplify.Storage.list(
path: StoragePath.fromString(uniquePrefix),
).result;

for (final uploadedPath in uploadedPaths) {
final listResultSecondaryBucket = await Amplify.Storage.list(
path: StoragePath.fromString(uniquePrefix),
options: StorageListOptions(
bucket: secondaryBucket,
),
).result;
for (var pathIndex = 0;
pathIndex < uploadedPaths.length ~/ 2;
pathIndex++) {
expect(
listResultMainBucket.items
.any((item) => item.path == uploadedPaths[pathIndex]),
isTrue,
);
}
for (var pathIndex = uploadedPaths.length ~/ 2;
pathIndex < uploadedPaths.length;
pathIndex++) {
expect(
listResult.items.any((item) => item.path == uploadedPath),
listResultSecondaryBucket.items
.any((item) => item.path == uploadedPaths[pathIndex]),
isTrue,
);
}
Expand Down Expand Up @@ -101,6 +142,17 @@ void main() {
),
).result as S3ListResult;

final listResultSecondaryBucket = await Amplify.Storage.list(
path: StoragePath.fromString('$uniquePrefix/'),
options: StorageListOptions(
pluginOptions: const S3ListPluginOptions(
excludeSubPaths: true,
delimiter: '#',
),
bucket: secondaryBucket,
),
).result as S3ListResult;

expect(listResult.items.length, 3);
expect(listResult.items.first.path, contains('file1.txt'));

Expand All @@ -110,6 +162,19 @@ void main() {
'$uniquePrefix/subdir2#',
);
expect(listResult.metadata.delimiter, '#');

expect(listResultSecondaryBucket.items.length, 3);
expect(
listResultSecondaryBucket.items.first.path,
contains('file5.txt'),
);

expect(listResultSecondaryBucket.metadata.subPaths.length, 1);
expect(
listResultSecondaryBucket.metadata.subPaths.first,
'$uniquePrefix/subdir4#',
);
expect(listResultSecondaryBucket.metadata.delimiter, '#');
});
});

Expand All @@ -123,6 +188,20 @@ void main() {

expect(listResult.items.length, 2);
expect(listResult.items.first.path, contains('file1.txt'));

final listResultSecondaryBucket = await Amplify.Storage.list(
path: StoragePath.fromString(uniquePrefix),
options: StorageListOptions(
pageSize: 2,
bucket: secondaryBucket,
),
).result;

expect(listResultSecondaryBucket.items.length, 2);
expect(
listResultSecondaryBucket.items.first.path,
contains('file5.txt'),
);
});

testWidgets('should list files with pagination', (_) async {
Expand Down Expand Up @@ -157,8 +236,22 @@ void main() {
),
).result;

expect(listResult.items.length, uploadedPaths.length);
expect(listResult.items.length, uploadedPaths.length ~/ 2);
expect(listResult.nextToken, isNull);

final listResultSecondaryBucket = await Amplify.Storage.list(
path: StoragePath.fromString(uniquePrefix),
options: StorageListOptions(
pluginOptions: const S3ListPluginOptions.listAll(),
bucket: secondaryBucket,
),
).result;

expect(
listResultSecondaryBucket.items.length,
uploadedPaths.length ~/ 2,
);
expect(listResultSecondaryBucket.nextToken, isNull);
});
});
});
Expand Down
1 change: 1 addition & 0 deletions packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface
final s3Options = StorageListOptions(
pluginOptions: s3PluginOptions,
nextToken: options?.nextToken,
bucket: options?.bucket,
pageSize: options?.pageSize ?? 1000,
);

Expand Down
9 changes: 5 additions & 4 deletions ...e/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,11 +125,12 @@ class StorageS3Service {
const S3ListPluginOptions();

final resolvedPath = await _pathResolver.resolvePath(path: path);
final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket);

if (!s3PluginOptions.listAll) {
final request = s3.ListObjectsV2Request.build((builder) {
builder
..bucket = _storageOutputs.bucketName
..bucket = s3ClientInfo.bucketName
..prefix = resolvedPath
..maxKeys = options.pageSize
..continuationToken = options.nextToken
Expand All @@ -140,7 +141,7 @@ class StorageS3Service {

try {
return S3ListResult.fromPaginatedResult(
await _defaultS3Client.listObjectsV2(request).result,
await s3ClientInfo.client.listObjectsV2(request).result,
);
} on smithy.UnknownSmithyHttpException catch (error) {
// S3Client.headObject may return 403 error
Expand All @@ -156,14 +157,14 @@ class StorageS3Service {
try {
final request = s3.ListObjectsV2Request.build((builder) {
builder
..bucket = _storageOutputs.bucketName
..bucket = s3ClientInfo.bucketName
..prefix = resolvedPath
..delimiter = s3PluginOptions.excludeSubPaths
? s3PluginOptions.delimiter
: null;
});

listResult = await _defaultS3Client.listObjectsV2(request).result;
listResult = await s3ClientInfo.client.listObjectsV2(request).result;
recursiveResult = S3ListResult.fromPaginatedResult(
listResult,
);
Expand Down
3 changes: 3 additions & 0 deletions packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,9 @@ void main() {
const testOptions = StorageListOptions(
pluginOptions: S3ListPluginOptions(excludeSubPaths: true),
nextToken: 'next-token-123',
bucket: StorageBucket.fromBucketInfo(
BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'),
),
pageSize: 2,
);

Expand Down

0 comments on commit 59ebae6

Please sign in to comment.