Skip to content

Commit adb2909

Browse files
mpivchevmpivchev
authored
Use multipartupload for uploading chunks to s3 (#2541)
* WIP Signed-off-by: Milen Pivchev <milen.pivchev@gmail.com> * Change stepper values Signed-off-by: Milen Pivchev <milen.pivchev@gmail.com> * Add check Signed-off-by: Milen Pivchev <milen.pivchev@gmail.com> * WIP Signed-off-by: Milen Pivchev <milen.pivchev@gmail.com> * Finish Signed-off-by: Milen Pivchev <milen.pivchev@gmail.com> --------- Signed-off-by: Milen Pivchev <milen.pivchev@gmail.com>
1 parent 8dbf5fc commit adb2909

File tree

5 files changed

+30
-18
lines changed

5 files changed

+30
-18
lines changed

‎iOSClient/Data/NCManageDatabase.swift

-1
Original file line numberDiff line numberDiff line change
@@ -350,7 +350,6 @@ class NCManageDatabase: NSObject {
350350
do {
351351
let realm = try Realm()
352352
try realm.write {
353-
354353
let result = realm.objects(tableChunk.self).filter(NSPredicate(format: "account == %@ AND ocId == %@", account, ocId))
355354
realm.delete(result)
356355
}

‎iOSClient/Networking/NCNetworkingChunkedUpload.swift

+20-12
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ extension NCNetworking {
4242
var uploadError = NKError()
4343

4444
var filesNames = NCManageDatabase.shared.getChunks(account: metadata.account, ocId: metadata.ocId)
45+
4546
if filesNames.count == 0 {
4647
filesNames = NextcloudKit.shared.nkCommonInstance.chunkedFile(inputDirectory: directoryProviderStorageOcId, outputDirectory: directoryProviderStorageOcId, fileName: metadata.fileName, chunkSizeMB: chunkSize)
4748
if filesNames.count > 0 {
@@ -57,7 +58,13 @@ extension NCNetworking {
5758
NotificationCenter.default.postOnMainThread(name: NCGlobal.shared.notificationCenterReloadDataSource, userInfo: ["serverUrl": metadata.serverUrl])
5859
}
5960

60-
createChunkedFolder(chunkFolderPath: chunkFolderPath, account: metadata.account) { error in
61+
let pathServerUrl = CCUtility.returnPathfromServerUrl(metadata.serverUrl, urlBase: metadata.urlBase, userId: metadata.userId, account: metadata.account)!
62+
let serverUrlFileNameDestination = metadata.urlBase + "/" + NextcloudKit.shared.nkCommonInstance.dav + "/files/" + metadata.userId + pathServerUrl + "/" + metadata.fileName
63+
64+
let destinationHeader: [String: String] = ["Destination" : serverUrlFileNameDestination]
65+
66+
// Create folder for chunks
67+
createChunkedFolder(customHeaders: destinationHeader, chunkFolderPath: chunkFolderPath, account: metadata.account) { error in
6168

6269
NCContentPresenter.shared.dismiss(after: NCGlobal.shared.dismissAfterSecond)
6370

@@ -71,6 +78,7 @@ extension NCNetworking {
7178

7279
start()
7380

81+
// Upload the chunks
7482
for fileName in filesNames {
7583

7684
let serverUrlFileName = chunkFolderPath + "/" + fileName
@@ -91,7 +99,7 @@ extension NCNetworking {
9199

92100
let semaphore = DispatchSemaphore(value: 0)
93101

94-
NextcloudKit.shared.upload(serverUrlFileName: serverUrlFileName, fileNameLocalPath: fileNameChunkLocalPath, requestHandler: { request in
102+
NextcloudKit.shared.upload(serverUrlFileName: serverUrlFileName, fileNameLocalPath: fileNameChunkLocalPath, addCustomHeaders: destinationHeader, requestHandler: { request in
95103

96104
self.uploadRequest[fileNameLocalPath] = request
97105

@@ -147,28 +155,28 @@ extension NCNetworking {
147155
return
148156
}
149157

150-
// Assembling the chunks
158+
// Assemble the chunks
151159
let serverUrlFileNameSource = chunkFolderPath + "/.file"
152-
let pathServerUrl = CCUtility.returnPathfromServerUrl(metadata.serverUrl, urlBase: metadata.urlBase, userId: metadata.userId, account: metadata.account)!
153-
let serverUrlFileNameDestination = metadata.urlBase + "/" + NextcloudKit.shared.nkCommonInstance.dav + "/files/" + metadata.userId + pathServerUrl + "/" + metadata.fileName
154160

155-
var customHeader: [String: String] = [:]
161+
var customHeaders: [String: String] = [:]
156162

157163
if metadata.creationDate.timeIntervalSince1970 > 0 {
158-
customHeader["X-OC-CTime"] = "\(metadata.creationDate.timeIntervalSince1970)"
164+
customHeaders["X-OC-CTime"] = "\(metadata.creationDate.timeIntervalSince1970)"
159165
}
160166

161167
if metadata.date.timeIntervalSince1970 > 0 {
162-
customHeader["X-OC-MTime"] = "\(metadata.date.timeIntervalSince1970)"
168+
customHeaders["X-OC-MTime"] = "\(metadata.date.timeIntervalSince1970)"
163169
}
164170

171+
destinationHeader.forEach { customHeaders[$0] = $1 }
172+
165173
// Calculate Assemble Timeout
166174
let ASSEMBLE_TIME_PER_GB: Double = 3 * 60 // 3 min
167175
let ASSEMBLE_TIME_MIN: Double = 60 // 60 sec
168176
let ASSEMBLE_TIME_MAX: Double = 30 * 60 // 30 min
169177
let timeout = max(ASSEMBLE_TIME_MIN, min(ASSEMBLE_TIME_PER_GB * fileSizeInGB, ASSEMBLE_TIME_MAX))
170178

171-
let options = NKRequestOptions(customHeader: customHeader, timeout: timeout, queue: DispatchQueue.global())
179+
let options = NKRequestOptions(customHeader: customHeaders, timeout: timeout, queue: DispatchQueue.global())
172180

173181
NextcloudKit.shared.moveFileOrFolder(serverUrlFileNameSource: serverUrlFileNameSource, serverUrlFileNameDestination: serverUrlFileNameDestination, overwrite: true, options: options) { _, error in
174182

@@ -223,10 +231,10 @@ extension NCNetworking {
223231
}
224232
}
225233

226-
private func createChunkedFolder(chunkFolderPath: String, account: String, completion: @escaping (_ errorCode: NKError) -> Void) {
234+
private func createChunkedFolder(customHeaders: [String : String], chunkFolderPath: String, account: String, completion: @escaping (_ errorCode: NKError) -> Void) {
235+
236+
let options = NKRequestOptions(customHeader: customHeaders, queue: NextcloudKit.shared.nkCommonInstance.backgroundQueue)
227237

228-
let options = NKRequestOptions(queue: NextcloudKit.shared.nkCommonInstance.backgroundQueue)
229-
230238
NextcloudKit.shared.readFileOrFolder(serverUrlFileName: chunkFolderPath, depth: "0", showHiddenFiles: CCUtility.getShowHiddenFiles(), options: options) { _, _, _, error in
231239

232240
if error == .success {

‎iOSClient/Settings/CCAdvanced.m

+2-3
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,8 @@ - (void)initializeForm
114114
[row.cellConfig setObject:[UIFont systemFontOfSize:15.0] forKey:@"textLabel.font"];
115115
[row.cellConfig setObject:UIColor.labelColor forKey:@"textLabel.textColor"];
116116
[row.cellConfigAtConfigure setObject:@YES forKey:@"stepControl.wraps"];
117-
[row.cellConfigAtConfigure setObject:@1 forKey:@"stepControl.stepValue"];
117+
118+
[row.cellConfigAtConfigure setObject:@10 forKey:@"stepControl.stepValue"];
118119
[row.cellConfigAtConfigure setObject:@0 forKey:@"stepControl.minimumValue"];
119120
[row.cellConfigAtConfigure setObject:@100 forKey:@"stepControl.maximumValue"];
120121
[section addFormRow:row];
@@ -254,7 +255,6 @@ - (void)initializeForm
254255
[XLFormOptionsObject formOptionsObjectWithValue:@(90) displayText:NSLocalizedString(@"_3_months_", nil)],
255256
[XLFormOptionsObject formOptionsObjectWithValue:@(30) displayText:NSLocalizedString(@"_1_month_", nil)],
256257
[XLFormOptionsObject formOptionsObjectWithValue:@(7) displayText:NSLocalizedString(@"_1_week_", nil)],
257-
//[XLFormOptionsObject formOptionsObjectWithValue:@(1) displayText:NSLocalizedString(@"_1_day_", nil)],
258258
];
259259
[sectionSize addFormRow:row];
260260

@@ -375,7 +375,6 @@ - (void)formRowDescriptorValueHasChanged:(XLFormRowDescriptor *)rowDescriptor ol
375375
}
376376

377377
if ([rowDescriptor.tag isEqualToString:@"chunk"]) {
378-
379378
NSInteger chunkSize = [[rowDescriptor.value valueData] intValue];
380379
[CCUtility setChunkSize:chunkSize];
381380
}

‎iOSClient/Supporting Files/en.lproj/Localizable.strings

+1
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@
156156
"_professional_" = "Professional";
157157
"_current_" = "Current";
158158
"_buy_" = "Buy";
159+
"_disabled_" = "Disabled";
159160

160161
/* MARK: Files lock */
161162

‎iOSClient/Utility/CCUtility.m

+7-2
Original file line numberDiff line numberDiff line change
@@ -667,19 +667,24 @@ + (void)setAccountRequest:(BOOL)set
667667
[UICKeyChainStore setString:sSet forKey:@"accountRequest" service:NCGlobal.shared.serviceShareKeyChain];
668668
}
669669

670+
/// In megabytes (MB)
670671
+ (NSInteger)getChunkSize
671672
{
672673
NSString *size = [UICKeyChainStore stringForKey:@"chunkSize" service:NCGlobal.shared.serviceShareKeyChain];
673-
674-
if (size == nil) {
674+
NSInteger sizeInt = [size integerValue];
675+
676+
if (size == nil || sizeInt < 10) {
675677
return 0;
676678
} else {
677679
return [size integerValue];
678680
}
679681
}
680682

683+
/// In megabytes (MB)
681684
+ (void)setChunkSize:(NSInteger)size
682685
{
686+
if (size < 10) size = 0;
687+
683688
NSString *sizeString = [@(size) stringValue];
684689
[UICKeyChainStore setString:sizeString forKey:@"chunkSize" service:NCGlobal.shared.serviceShareKeyChain];
685690
}

0 commit comments

Comments
 (0)