Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(storage): multi bucket upload file #5600

Merged
merged 3 commits into from
Oct 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

import 'package:aws_common/aws_common.dart';
import 'package:amplify_core/amplify_core.dart';

/// {@template amplify_core.storage.upload_file_options}
/// Configurable options for `Amplify.Storage.uploadFile`.
Expand All @@ -15,6 +15,7 @@ class StorageUploadFileOptions
const StorageUploadFileOptions({
this.metadata = const {},
this.pluginOptions,
this.bucket,
});

/// The metadata attached to the object to be uploaded.
Expand All @@ -23,8 +24,11 @@ class StorageUploadFileOptions
/// {@macro amplify_core.storage.upload_file_plugin_options}
final StorageUploadFilePluginOptions? pluginOptions;

/// Optionally specify which bucket to target
final StorageBucket? bucket;

@override
List<Object?> get props => [metadata, pluginOptions];
List<Object?> get props => [metadata, pluginOptions, bucket];

@override
String get runtimeTypeName => 'StorageUploadFileOptions';
Expand All @@ -33,6 +37,7 @@ class StorageUploadFileOptions
Map<String, Object?> toJson() => {
'metadata': metadata,
'pluginOptions': pluginOptions?.toJson(),
'bucket': bucket,
};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,18 +136,39 @@ void main() {
testWidgets('multi bucket', (_) async {
final mainBucket =
StorageBucket.fromOutputs('Storage Integ Test main bucket');
final secondaryBucket = StorageBucket.fromOutputs(
'Storage Integ Test secondary bucket',
);
await Amplify.Storage.uploadData(
path: StoragePath.fromString(publicPath),
data: StorageDataPayload.bytes(bytesData),
bucket: secondaryBucket,
).result;

// TODO(equartey): Add download check for secondary bucket when upload supports multibucket
final downloadResult = await Amplify.Storage.downloadData(
path: StoragePath.fromIdentityId(
(identityId) => 'private/$identityId/$identityName',
),
path: StoragePath.fromString(publicPath),
options: StorageDownloadDataOptions(bucket: mainBucket),
).result;
expect(downloadResult.bytes, identityData);
expect(
downloadResult.bytes,
bytesData,
);
expect(
downloadResult.downloadedItem.path,
'private/$userIdentityId/$identityName',
publicPath,
);

final downloadSecondaryResult = await Amplify.Storage.downloadData(
path: StoragePath.fromString(publicPath),
options: StorageDownloadDataOptions(bucket: secondaryBucket),
).result;
expect(
downloadSecondaryResult.bytes,
bytesData,
);
expect(
downloadSecondaryResult.downloadedItem.path,
publicPath,
);
});
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import 'package:integration_test/integration_test.dart';

import 'utils/configure.dart';
import 'utils/create_file/create_file.dart';
import 'utils/object_exists.dart';
import 'utils/sign_in_new_user.dart';
import 'utils/tear_down.dart';

Expand Down Expand Up @@ -220,6 +221,83 @@ void main() {
});
});

group('multi-bucket', () {
final mainBucket =
StorageBucket.fromOutputs('Storage Integ Test main bucket');
final secondaryBucket = StorageBucket.fromOutputs(
'Storage Integ Test secondary bucket',
);

testWidgets('uploads to multiple buckets', (_) async {
final fileId = uuid();
final path = 'public/multi-bucket-upload-file-$fileId';
final storagePath = StoragePath.fromString(path);
const content = 'upload file';
final data = content.codeUnits;
final filePath = await createFile(path: fileId, content: content);
addTearDownMultiBucket(
storagePath,
[mainBucket, secondaryBucket],
);
// main bucket
final mainResult = await Amplify.Storage.uploadFile(
localFile: AWSFile.fromPath(filePath),
path: storagePath,
options: StorageUploadFileOptions(
pluginOptions: const S3UploadFilePluginOptions(
useAccelerateEndpoint: true,
),
bucket: mainBucket,
),
).result;
expect(mainResult.uploadedItem.path, path);

final downloadMainResult = await Amplify.Storage.downloadData(
path: storagePath,
options: StorageDownloadDataOptions(
bucket: mainBucket,
),
).result;
expect(downloadMainResult.bytes, data);

// secondary bucket
final secondaryResult = await Amplify.Storage.uploadFile(
localFile: AWSFile.fromPath(filePath),
path: storagePath,
options: StorageUploadFileOptions(
pluginOptions: const S3UploadFilePluginOptions(
useAccelerateEndpoint: true,
),
bucket: secondaryBucket,
),
).result;
expect(secondaryResult.uploadedItem.path, path);

final downloadSecondaryResult = await Amplify.Storage.downloadData(
path: storagePath,
options: StorageDownloadDataOptions(
bucket: secondaryBucket,
),
).result;
expect(downloadSecondaryResult.bytes, data);

expect(
await objectExists(
storagePath,
bucket: mainBucket,
),
true,
);
expect(
await objectExists(
storagePath,
bucket: secondaryBucket,
),
true,
);
});
});

group('upload progress', () {
testWidgets('reports progress', (_) async {
final fileId = uuid();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@ import 'package:flutter_test/flutter_test.dart';
final _logger = AmplifyLogger().createChild('StorageTests');

/// Adds a tear down to remove the object at [path].
void addTearDownPath(StoragePath path) {
void addTearDownPath(StoragePath path, {StorageBucket? bucket}) {
addTearDown(
() {
try {
return Amplify.Storage.remove(path: path).result;
return Amplify.Storage.remove(
path: path,
options: StorageRemoveOptions(bucket: bucket),
).result;
} on Exception catch (e) {
_logger.warn('Failed to remove file after test', e);
rethrow;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface
final s3Options = StorageUploadFileOptions(
metadata: options?.metadata ?? const {},
pluginOptions: s3PluginOptions,
bucket: options?.bucket,
);

final uploadTask = storageS3Service.uploadFile(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,7 @@ class StorageS3Service {
FutureOr<void> Function()? onDone,
FutureOr<void> Function()? onError,
}) {
final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket);
final s3PluginOptions =
options.pluginOptions as S3UploadFilePluginOptions? ??
const S3UploadFilePluginOptions();
Expand All @@ -374,9 +375,9 @@ class StorageS3Service {
);
final uploadDataTask = S3UploadTask.fromAWSFile(
localFile,
s3Client: _defaultS3Client,
s3ClientConfig: _defaultS3ClientConfig,
bucket: _storageOutputs.bucketName,
s3Client: s3ClientInfo.client,
s3ClientConfig: s3ClientInfo.config,
bucket: s3ClientInfo.bucketName,
awsRegion: _storageOutputs.awsRegion,
path: path,
options: uploadDataOptions,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,9 @@ void main() {
() async {
const defaultOptions = StorageDownloadDataOptions(
pluginOptions: S3DownloadDataPluginOptions(),
bucket: StorageBucket.fromBucketInfo(
BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'),
),
);

when(
Expand All @@ -432,6 +435,7 @@ void main() {

downloadDataOperation = storageS3Plugin.downloadData(
path: const StoragePath.fromString('public/$testKey'),
options: defaultOptions,
);

final capturedOptions = verify(
Expand Down Expand Up @@ -766,6 +770,9 @@ void main() {
() async {
const defaultOptions = StorageUploadFileOptions(
pluginOptions: S3UploadFilePluginOptions(),
bucket: StorageBucket.fromBucketInfo(
BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'),
),
);

when(
Expand All @@ -784,6 +791,7 @@ void main() {
uploadFileOperation = storageS3Plugin.uploadFile(
path: testPath,
localFile: testLocalFile,
options: defaultOptions,
);

final capturedParams = verify(
Expand Down Expand Up @@ -1009,6 +1017,9 @@ void main() {
() async {
const defaultOptions = StorageRemoveOptions(
pluginOptions: S3RemovePluginOptions(),
bucket: StorageBucket.fromBucketInfo(
BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'),
),
);
when(
() => storageS3Service.remove(
Expand All @@ -1017,7 +1028,10 @@ void main() {
),
).thenAnswer((_) async => testResult);

final removeOperation = storageS3Plugin.remove(path: testPath);
final removeOperation = storageS3Plugin.remove(
path: testPath,
options: defaultOptions,
);

final capturedOptions = verify(
() => storageS3Service.remove(
Expand Down
Loading