From 4e78ed4179f2720b6fffca82909499ad6d83b801 Mon Sep 17 00:00:00 2001 From: Elijah Quartey Date: Thu, 24 Oct 2024 10:37:30 -0500 Subject: [PATCH 1/3] feat(storage): multi bucket upload file --- .../types/storage/upload_file_options.dart | 9 ++- .../integration_test/upload_file_test.dart | 71 +++++++++++++++++++ .../service/storage_s3_service_impl.dart | 7 +- 3 files changed, 82 insertions(+), 5 deletions(-) diff --git a/packages/amplify_core/lib/src/types/storage/upload_file_options.dart b/packages/amplify_core/lib/src/types/storage/upload_file_options.dart index bad1529468..6b025257fb 100644 --- a/packages/amplify_core/lib/src/types/storage/upload_file_options.dart +++ b/packages/amplify_core/lib/src/types/storage/upload_file_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.upload_file_options} /// Configurable options for `Amplify.Storage.uploadFile`. @@ -15,6 +15,7 @@ class StorageUploadFileOptions const StorageUploadFileOptions({ this.metadata = const {}, this.pluginOptions, + this.bucket, }); /// The metadata attached to the object to be uploaded. @@ -23,8 +24,11 @@ class StorageUploadFileOptions /// {@macro amplify_core.storage.upload_file_plugin_options} final StorageUploadFilePluginOptions? pluginOptions; + /// Optionally specify which bucket to target + final StorageBucket? bucket; + @override - List get props => [metadata, pluginOptions]; + List get props => [metadata, pluginOptions, bucket]; @override String get runtimeTypeName => 'StorageUploadFileOptions'; @@ -33,6 +37,7 @@ class StorageUploadFileOptions Map toJson() => { 'metadata': metadata, 'pluginOptions': pluginOptions?.toJson(), + 'bucket': bucket, }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart index 6e1eb0581e..f417735db1 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart @@ -11,6 +11,7 @@ import 'package:integration_test/integration_test.dart'; import 'utils/configure.dart'; import 'utils/create_file/create_file.dart'; +import 'utils/object_exists.dart'; import 'utils/sign_in_new_user.dart'; import 'utils/tear_down.dart'; @@ -220,6 +221,76 @@ void main() { }); }); + group('multi-bucket', () { + final mainBucket = + StorageBucket.fromOutputs('Storage Integ Test main bucket'); + final secondaryBucket = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); + + testWidgets('uploads to multiple buckets', (_) async { + final fileId = uuid(); + final path = + StoragePath.fromString('public/multi-bucket-upload-file-$fileId'); + const content = 'upload file'; + final data = content.codeUnits; + final filePath = await createFile(path: fileId, content: content); + addTearDownMultiBucket( + path, + [mainBucket, secondaryBucket], + ); + // main bucket + final mainResult = await Amplify.Storage.uploadFile( + localFile: AWSFile.fromPath(filePath), + path: path, + options: StorageUploadFileOptions( + pluginOptions: const S3UploadFilePluginOptions( + useAccelerateEndpoint: true, + ), + bucket: mainBucket, + ), + ).result; + expect(mainResult.uploadedItem.path, path); + + final downloadMainResult = await Amplify.Storage.downloadData( + path: path, + options: StorageDownloadDataOptions( + bucket: mainBucket, + ), + ).result; + expect(downloadMainResult.bytes, data); + + // secondary bucket + final secondaryResult = await Amplify.Storage.uploadFile( + localFile: AWSFile.fromPath(filePath), + path: path, + options: StorageUploadFileOptions( + pluginOptions: const S3UploadFilePluginOptions( + useAccelerateEndpoint: true, + ), + bucket: secondaryBucket, + ), + ).result; + expect(secondaryResult.uploadedItem.path, path); + + final downloadSecondaryResult = await Amplify.Storage.downloadData( + path: path, + options: StorageDownloadDataOptions( + bucket: secondaryBucket, + ), + ).result; + expect(downloadSecondaryResult.bytes, data); + + expect( + await objectExistsInBuckets( + path, + [mainBucket, secondaryBucket], + ), + true, + ); + }); + }); + group('upload progress', () { testWidgets('reports progress', (_) async { final fileId = uuid(); diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 72fe242f80..4ef4417ae8 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -362,6 +362,7 @@ class StorageS3Service { FutureOr Function()? onDone, FutureOr Function()? onError, }) { + final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket); final s3PluginOptions = options.pluginOptions as S3UploadFilePluginOptions? ?? const S3UploadFilePluginOptions(); @@ -374,9 +375,9 @@ class StorageS3Service { ); final uploadDataTask = S3UploadTask.fromAWSFile( localFile, - s3Client: _defaultS3Client, - s3ClientConfig: _defaultS3ClientConfig, - bucket: _storageOutputs.bucketName, + s3Client: s3ClientInfo.client, + s3ClientConfig: s3ClientInfo.config, + bucket: s3ClientInfo.bucketName, awsRegion: _storageOutputs.awsRegion, path: path, options: uploadDataOptions, From ee0bac0a45d0d3ea62b8ee6704e1f1cdd230d4a1 Mon Sep 17 00:00:00 2001 From: Elijah Quartey Date: Fri, 25 Oct 2024 13:11:59 -0500 Subject: [PATCH 2/3] fixed missing param --- .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index 24bc55dcc2..3b66d88145 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -324,6 +324,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageUploadFileOptions( metadata: options?.metadata ?? const {}, pluginOptions: s3PluginOptions, + bucket: options?.bucket, ); final uploadTask = storageS3Service.uploadFile( From 530140a83a3aeca0f2bae7278cd0ab5df5ba1f91 Mon Sep 17 00:00:00 2001 From: Elijah Quartey Date: Mon, 28 Oct 2024 15:01:22 -0500 Subject: [PATCH 3/3] fixing up tests --- .../integration_test/download_data_test.dart | 33 +++++++++++++++---- .../integration_test/upload_file_test.dart | 27 +++++++++------ .../integration_test/utils/tear_down.dart | 7 ++-- .../test/amplify_storage_s3_dart_test.dart | 16 ++++++++- 4 files changed, 64 insertions(+), 19 deletions(-) diff --git a/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart index a88c3631aa..96dd7c7c36 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart @@ -136,18 +136,39 @@ void main() { testWidgets('multi bucket', (_) async { final mainBucket = StorageBucket.fromOutputs('Storage Integ Test main bucket'); + final secondaryBucket = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); + await Amplify.Storage.uploadData( + path: StoragePath.fromString(publicPath), + data: StorageDataPayload.bytes(bytesData), + bucket: secondaryBucket, + ).result; - // TODO(equartey): Add download check for secondary bucket when upload supports multibucket final downloadResult = await Amplify.Storage.downloadData( - path: StoragePath.fromIdentityId( - (identityId) => 'private/$identityId/$identityName', - ), + path: StoragePath.fromString(publicPath), options: StorageDownloadDataOptions(bucket: mainBucket), ).result; - expect(downloadResult.bytes, identityData); + expect( + downloadResult.bytes, + bytesData, + ); expect( downloadResult.downloadedItem.path, - 'private/$userIdentityId/$identityName', + publicPath, + ); + + final downloadSecondaryResult = await Amplify.Storage.downloadData( + path: StoragePath.fromString(publicPath), + options: StorageDownloadDataOptions(bucket: secondaryBucket), + ).result; + expect( + downloadSecondaryResult.bytes, + bytesData, + ); + expect( + downloadSecondaryResult.downloadedItem.path, + publicPath, ); }); }); diff --git a/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart index f417735db1..9113a3c4c3 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart @@ -230,19 +230,19 @@ void main() { testWidgets('uploads to multiple buckets', (_) async { final fileId = uuid(); - final path = - StoragePath.fromString('public/multi-bucket-upload-file-$fileId'); + final path = 'public/multi-bucket-upload-file-$fileId'; + final storagePath = StoragePath.fromString(path); const content = 'upload file'; final data = content.codeUnits; final filePath = await createFile(path: fileId, content: content); addTearDownMultiBucket( - path, + storagePath, [mainBucket, secondaryBucket], ); // main bucket final mainResult = await Amplify.Storage.uploadFile( localFile: AWSFile.fromPath(filePath), - path: path, + path: storagePath, options: StorageUploadFileOptions( pluginOptions: const S3UploadFilePluginOptions( useAccelerateEndpoint: true, @@ -253,7 +253,7 @@ void main() { expect(mainResult.uploadedItem.path, path); final downloadMainResult = await Amplify.Storage.downloadData( - path: path, + path: storagePath, options: StorageDownloadDataOptions( bucket: mainBucket, ), @@ -263,7 +263,7 @@ void main() { // secondary bucket final secondaryResult = await Amplify.Storage.uploadFile( localFile: AWSFile.fromPath(filePath), - path: path, + path: storagePath, options: StorageUploadFileOptions( pluginOptions: const S3UploadFilePluginOptions( useAccelerateEndpoint: true, @@ -274,7 +274,7 @@ void main() { expect(secondaryResult.uploadedItem.path, path); final downloadSecondaryResult = await Amplify.Storage.downloadData( - path: path, + path: storagePath, options: StorageDownloadDataOptions( bucket: secondaryBucket, ), @@ -282,9 +282,16 @@ void main() { expect(downloadSecondaryResult.bytes, data); expect( - await objectExistsInBuckets( - path, - [mainBucket, secondaryBucket], + await objectExists( + storagePath, + bucket: mainBucket, + ), + true, + ); + expect( + await objectExists( + storagePath, + bucket: secondaryBucket, ), true, ); diff --git a/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart b/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart index 319cc05ee9..21cddc0675 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart @@ -7,11 +7,14 @@ import 'package:flutter_test/flutter_test.dart'; final _logger = AmplifyLogger().createChild('StorageTests'); /// Adds a tear down to remove the object at [path]. -void addTearDownPath(StoragePath path) { +void addTearDownPath(StoragePath path, {StorageBucket? bucket}) { addTearDown( () { try { - return Amplify.Storage.remove(path: path).result; + return Amplify.Storage.remove( + path: path, + options: StorageRemoveOptions(bucket: bucket), + ).result; } on Exception catch (e) { _logger.warn('Failed to remove file after test', e); rethrow; diff --git a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart index 3968db24ed..6e2d258b90 100644 --- a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart @@ -415,6 +415,9 @@ void main() { () async { const defaultOptions = StorageDownloadDataOptions( pluginOptions: S3DownloadDataPluginOptions(), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( @@ -432,6 +435,7 @@ void main() { downloadDataOperation = storageS3Plugin.downloadData( path: const StoragePath.fromString('public/$testKey'), + options: defaultOptions, ); final capturedOptions = verify( @@ -766,6 +770,9 @@ void main() { () async { const defaultOptions = StorageUploadFileOptions( pluginOptions: S3UploadFilePluginOptions(), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( @@ -784,6 +791,7 @@ void main() { uploadFileOperation = storageS3Plugin.uploadFile( path: testPath, localFile: testLocalFile, + options: defaultOptions, ); final capturedParams = verify( @@ -1009,6 +1017,9 @@ void main() { () async { const defaultOptions = StorageRemoveOptions( pluginOptions: S3RemovePluginOptions(), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( () => storageS3Service.remove( @@ -1017,7 +1028,10 @@ void main() { ), ).thenAnswer((_) async => testResult); - final removeOperation = storageS3Plugin.remove(path: testPath); + final removeOperation = storageS3Plugin.remove( + path: testPath, + options: defaultOptions, + ); final capturedOptions = verify( () => storageS3Service.remove(