Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use multipartupload for uploading chunks to s3 #2541

Merged
merged 7 commits into from
Jul 27, 2023
Merged

Use multipartupload for uploading chunks to s3 #2541

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b3c51fe
WIP
mpivchev Jul 19, 2023
cbc36b6
Change stepper values
mpivchev Jul 19, 2023
191d401
Add check
mpivchev Jul 19, 2023
a407ba6
WIP
mpivchev Jul 21, 2023
1d9b218
Merge branch 'develop' of https://github.com/nextcloud/ios into 2471-…
mpivchev Jul 25, 2023
873ce8e
Finish
mpivchev Jul 25, 2023
3f13bd9
Merge branch 'develop' of https://github.com/nextcloud/ios into 2471-…
mpivchev Jul 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion iOSClient/Data/NCManageDatabase.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,6 @@ class NCManageDatabase: NSObject {
do {
let realm = try Realm()
try realm.write {

let result = realm.objects(tableChunk.self).filter(NSPredicate(format: "account == %@ AND ocId == %@", account, ocId))
realm.delete(result)
}
Expand Down
32 changes: 20 additions & 12 deletions iOSClient/Networking/NCNetworkingChunkedUpload.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ extension NCNetworking {
var uploadError = NKError()

var filesNames = NCManageDatabase.shared.getChunks(account: metadata.account, ocId: metadata.ocId)

if filesNames.count == 0 {
filesNames = NextcloudKit.shared.nkCommonInstance.chunkedFile(inputDirectory: directoryProviderStorageOcId, outputDirectory: directoryProviderStorageOcId, fileName: metadata.fileName, chunkSizeMB: chunkSize)
if filesNames.count > 0 {
Expand All @@ -57,7 +58,13 @@ extension NCNetworking {
NotificationCenter.default.postOnMainThread(name: NCGlobal.shared.notificationCenterReloadDataSource, userInfo: ["serverUrl": metadata.serverUrl])
}

createChunkedFolder(chunkFolderPath: chunkFolderPath, account: metadata.account) { error in
let pathServerUrl = CCUtility.returnPathfromServerUrl(metadata.serverUrl, urlBase: metadata.urlBase, userId: metadata.userId, account: metadata.account)!
let serverUrlFileNameDestination = metadata.urlBase + "/" + NextcloudKit.shared.nkCommonInstance.dav + "/files/" + metadata.userId + pathServerUrl + "/" + metadata.fileName

let destinationHeader: [String: String] = ["Destination" : serverUrlFileNameDestination]

// Create folder for chunks
createChunkedFolder(customHeaders: destinationHeader, chunkFolderPath: chunkFolderPath, account: metadata.account) { error in

NCContentPresenter.shared.dismiss(after: NCGlobal.shared.dismissAfterSecond)

Expand All @@ -71,6 +78,7 @@ extension NCNetworking {

start()

// Upload the chunks
for fileName in filesNames {

let serverUrlFileName = chunkFolderPath + "/" + fileName
Expand All @@ -91,7 +99,7 @@ extension NCNetworking {

let semaphore = DispatchSemaphore(value: 0)

NextcloudKit.shared.upload(serverUrlFileName: serverUrlFileName, fileNameLocalPath: fileNameChunkLocalPath, requestHandler: { request in
NextcloudKit.shared.upload(serverUrlFileName: serverUrlFileName, fileNameLocalPath: fileNameChunkLocalPath, addCustomHeaders: destinationHeader, requestHandler: { request in

self.uploadRequest[fileNameLocalPath] = request

Expand Down Expand Up @@ -147,28 +155,28 @@ extension NCNetworking {
return
}

// Assembling the chunks
// Assemble the chunks
let serverUrlFileNameSource = chunkFolderPath + "/.file"
let pathServerUrl = CCUtility.returnPathfromServerUrl(metadata.serverUrl, urlBase: metadata.urlBase, userId: metadata.userId, account: metadata.account)!
let serverUrlFileNameDestination = metadata.urlBase + "/" + NextcloudKit.shared.nkCommonInstance.dav + "/files/" + metadata.userId + pathServerUrl + "/" + metadata.fileName

var customHeader: [String: String] = [:]
var customHeaders: [String: String] = [:]

if metadata.creationDate.timeIntervalSince1970 > 0 {
customHeader["X-OC-CTime"] = "\(metadata.creationDate.timeIntervalSince1970)"
customHeaders["X-OC-CTime"] = "\(metadata.creationDate.timeIntervalSince1970)"
}

if metadata.date.timeIntervalSince1970 > 0 {
customHeader["X-OC-MTime"] = "\(metadata.date.timeIntervalSince1970)"
customHeaders["X-OC-MTime"] = "\(metadata.date.timeIntervalSince1970)"
}

destinationHeader.forEach { customHeaders[$0] = $1 }

// Calculate Assemble Timeout
let ASSEMBLE_TIME_PER_GB: Double = 3 * 60 // 3 min
let ASSEMBLE_TIME_MIN: Double = 60 // 60 sec
let ASSEMBLE_TIME_MAX: Double = 30 * 60 // 30 min
let timeout = max(ASSEMBLE_TIME_MIN, min(ASSEMBLE_TIME_PER_GB * fileSizeInGB, ASSEMBLE_TIME_MAX))

let options = NKRequestOptions(customHeader: customHeader, timeout: timeout, queue: DispatchQueue.global())
let options = NKRequestOptions(customHeader: customHeaders, timeout: timeout, queue: DispatchQueue.global())

NextcloudKit.shared.moveFileOrFolder(serverUrlFileNameSource: serverUrlFileNameSource, serverUrlFileNameDestination: serverUrlFileNameDestination, overwrite: true, options: options) { _, error in

Expand Down Expand Up @@ -223,10 +231,10 @@ extension NCNetworking {
}
}

private func createChunkedFolder(chunkFolderPath: String, account: String, completion: @escaping (_ errorCode: NKError) -> Void) {
private func createChunkedFolder(customHeaders: [String : String], chunkFolderPath: String, account: String, completion: @escaping (_ errorCode: NKError) -> Void) {

let options = NKRequestOptions(customHeader: customHeaders, queue: NextcloudKit.shared.nkCommonInstance.backgroundQueue)

let options = NKRequestOptions(queue: NextcloudKit.shared.nkCommonInstance.backgroundQueue)

NextcloudKit.shared.readFileOrFolder(serverUrlFileName: chunkFolderPath, depth: "0", showHiddenFiles: CCUtility.getShowHiddenFiles(), options: options) { _, _, _, error in

if error == .success {
Expand Down
5 changes: 2 additions & 3 deletions iOSClient/Settings/CCAdvanced.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@ - (void)initializeForm
[row.cellConfig setObject:[UIFont systemFontOfSize:15.0] forKey:@"textLabel.font"];
[row.cellConfig setObject:UIColor.labelColor forKey:@"textLabel.textColor"];
[row.cellConfigAtConfigure setObject:@YES forKey:@"stepControl.wraps"];
[row.cellConfigAtConfigure setObject:@1 forKey:@"stepControl.stepValue"];

[row.cellConfigAtConfigure setObject:@10 forKey:@"stepControl.stepValue"];
[row.cellConfigAtConfigure setObject:@0 forKey:@"stepControl.minimumValue"];
[row.cellConfigAtConfigure setObject:@100 forKey:@"stepControl.maximumValue"];
[section addFormRow:row];
Expand Down Expand Up @@ -254,7 +255,6 @@ - (void)initializeForm
[XLFormOptionsObject formOptionsObjectWithValue:@(90) displayText:NSLocalizedString(@"_3_months_", nil)],
[XLFormOptionsObject formOptionsObjectWithValue:@(30) displayText:NSLocalizedString(@"_1_month_", nil)],
[XLFormOptionsObject formOptionsObjectWithValue:@(7) displayText:NSLocalizedString(@"_1_week_", nil)],
//[XLFormOptionsObject formOptionsObjectWithValue:@(1) displayText:NSLocalizedString(@"_1_day_", nil)],
];
[sectionSize addFormRow:row];

Expand Down Expand Up @@ -375,7 +375,6 @@ - (void)formRowDescriptorValueHasChanged:(XLFormRowDescriptor *)rowDescriptor ol
}

if ([rowDescriptor.tag isEqualToString:@"chunk"]) {

NSInteger chunkSize = [[rowDescriptor.value valueData] intValue];
[CCUtility setChunkSize:chunkSize];
}
Expand Down
1 change: 1 addition & 0 deletions iOSClient/Supporting Files/en.lproj/Localizable.strings
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@
"_professional_" = "Professional";
"_current_" = "Current";
"_buy_" = "Buy";
"_disabled_" = "Disabled";

/* MARK: Files lock */

Expand Down
9 changes: 7 additions & 2 deletions iOSClient/Utility/CCUtility.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -667,19 +667,24 @@ + (void)setAccountRequest:(BOOL)set
[UICKeyChainStore setString:sSet forKey:@"accountRequest" service:NCGlobal.shared.serviceShareKeyChain];
}

/// In megabytes (MB)
+ (NSInteger)getChunkSize
{
NSString *size = [UICKeyChainStore stringForKey:@"chunkSize" service:NCGlobal.shared.serviceShareKeyChain];

if (size == nil) {
NSInteger sizeInt = [size integerValue];

if (size == nil || sizeInt < 10) {
return 0;
} else {
return [size integerValue];
}
}

/// In megabytes (MB)
+ (void)setChunkSize:(NSInteger)size
{
if (size < 10) size = 0;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the size should still be default to the minimum supported value of 5MB then if configured to be > 0.

Maybe there is some byte calculation in another place where the 5 MB -> bytes generates the wrong value?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can remove this two function and use the new algorithm

Comment on lines -673 to +686
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Int checks are temporary until we add auto chunking


NSString *sizeString = [@(size) stringValue];
[UICKeyChainStore setString:sizeString forKey:@"chunkSize" service:NCGlobal.shared.serviceShareKeyChain];
}
Expand Down