Skip to content

Commit b69fb7d

Browse files
tsdicloudilya.dokshukin@iits-consulting.de
authored and
ilya.dokshukin@iits-consulting.de
committed
S3 direct multipart upload optimisation.
The change was needed to avoid doubling of upload time as the original solution does a complete re-read of the S3 object on final MOVE of NextCloud chunked upload. See nextcloud#27034 for details. Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>
1 parent 1eea64f commit b69fb7d

23 files changed

+680
-20
lines changed

apps/dav/composer/composer/autoload_classmap.php

+2
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,7 @@
271271
'OCA\\DAV\\Search\\EventsSearchProvider' => $baseDir . '/../lib/Search/EventsSearchProvider.php',
272272
'OCA\\DAV\\Search\\TasksSearchProvider' => $baseDir . '/../lib/Search/TasksSearchProvider.php',
273273
'OCA\\DAV\\Server' => $baseDir . '/../lib/Server.php',
274+
'OCA\\DAV\\Service\\CustomPropertiesService' => $baseDir . '/../lib/Service/CustomPropertiesService.php',
274275
'OCA\\DAV\\Settings\\CalDAVSettings' => $baseDir . '/../lib/Settings/CalDAVSettings.php',
275276
'OCA\\DAV\\Storage\\PublicOwnerWrapper' => $baseDir . '/../lib/Storage/PublicOwnerWrapper.php',
276277
'OCA\\DAV\\SystemTag\\SystemTagMappingNode' => $baseDir . '/../lib/SystemTag/SystemTagMappingNode.php',
@@ -283,6 +284,7 @@
283284
'OCA\\DAV\\Traits\\PrincipalProxyTrait' => $baseDir . '/../lib/Traits/PrincipalProxyTrait.php',
284285
'OCA\\DAV\\Upload\\AssemblyStream' => $baseDir . '/../lib/Upload/AssemblyStream.php',
285286
'OCA\\DAV\\Upload\\ChunkingPlugin' => $baseDir . '/../lib/Upload/ChunkingPlugin.php',
287+
'OCA\\DAV\\Upload\\ChunkingV2Plugin' => $baseDir . '/../lib/Upload/ChunkingV2Plugin.php',
286288
'OCA\\DAV\\Upload\\CleanupService' => $baseDir . '/../lib/Upload/CleanupService.php',
287289
'OCA\\DAV\\Upload\\FutureFile' => $baseDir . '/../lib/Upload/FutureFile.php',
288290
'OCA\\DAV\\Upload\\RootCollection' => $baseDir . '/../lib/Upload/RootCollection.php',

apps/dav/composer/composer/autoload_static.php

+2
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ class ComposerStaticInitDAV
286286
'OCA\\DAV\\Search\\EventsSearchProvider' => __DIR__ . '/..' . '/../lib/Search/EventsSearchProvider.php',
287287
'OCA\\DAV\\Search\\TasksSearchProvider' => __DIR__ . '/..' . '/../lib/Search/TasksSearchProvider.php',
288288
'OCA\\DAV\\Server' => __DIR__ . '/..' . '/../lib/Server.php',
289+
'OCA\\DAV\\Service\\CustomPropertiesService' => __DIR__ . '/..' . '/../lib/Service/CustomPropertiesService.php',
289290
'OCA\\DAV\\Settings\\CalDAVSettings' => __DIR__ . '/..' . '/../lib/Settings/CalDAVSettings.php',
290291
'OCA\\DAV\\Storage\\PublicOwnerWrapper' => __DIR__ . '/..' . '/../lib/Storage/PublicOwnerWrapper.php',
291292
'OCA\\DAV\\SystemTag\\SystemTagMappingNode' => __DIR__ . '/..' . '/../lib/SystemTag/SystemTagMappingNode.php',
@@ -298,6 +299,7 @@ class ComposerStaticInitDAV
298299
'OCA\\DAV\\Traits\\PrincipalProxyTrait' => __DIR__ . '/..' . '/../lib/Traits/PrincipalProxyTrait.php',
299300
'OCA\\DAV\\Upload\\AssemblyStream' => __DIR__ . '/..' . '/../lib/Upload/AssemblyStream.php',
300301
'OCA\\DAV\\Upload\\ChunkingPlugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingPlugin.php',
302+
'OCA\\DAV\\Upload\\ChunkingV2Plugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingV2Plugin.php',
301303
'OCA\\DAV\\Upload\\CleanupService' => __DIR__ . '/..' . '/../lib/Upload/CleanupService.php',
302304
'OCA\\DAV\\Upload\\FutureFile' => __DIR__ . '/..' . '/../lib/Upload/FutureFile.php',
303305
'OCA\\DAV\\Upload\\RootCollection' => __DIR__ . '/..' . '/../lib/Upload/RootCollection.php',

apps/dav/lib/BackgroundJob/UploadCleanup.php

+9-1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
namespace OCA\DAV\BackgroundJob;
2929

3030
use OC\User\NoUserException;
31+
use OCA\DAV\Service\CustomPropertiesService;
3132
use OCP\AppFramework\Utility\ITimeFactory;
3233
use OCP\BackgroundJob\IJobList;
3334
use OCP\BackgroundJob\TimedJob;
@@ -44,10 +45,14 @@ class UploadCleanup extends TimedJob {
4445
/** @var IJobList */
4546
private $jobList;
4647

47-
public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList) {
48+
/** @var CustomPropertiesService */
49+
private $customPropertiesService;
50+
51+
public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList, CustomPropertiesService $customPropertiesService) {
4852
parent::__construct($time);
4953
$this->rootFolder = $rootFolder;
5054
$this->jobList = $jobList;
55+
$this->customPropertiesService = $customPropertiesService;
5156

5257
// Run once a day
5358
$this->setInterval(60 * 60 * 24);
@@ -71,6 +76,8 @@ protected function run($argument) {
7176

7277
$files = $uploadFolder->getDirectoryListing();
7378

79+
$davPath = 'uploads/' . $uid . '/' . $uploadFolder->getName();
80+
7481
// Remove if all files have an mtime of more than a day
7582
$time = $this->time->getTime() - 60 * 60 * 24;
7683

@@ -82,6 +89,7 @@ protected function run($argument) {
8289
}, $initial);
8390

8491
if ($expire) {
92+
$this->customPropertiesService->delete($uid, $davPath);
8593
$uploadFolder->delete();
8694
$this->jobList->remove(self::class, $argument);
8795
}

apps/dav/lib/Connector/Sabre/Directory.php

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
use OCA\DAV\Connector\Sabre\Exception\FileLocked;
3838
use OCA\DAV\Connector\Sabre\Exception\Forbidden;
3939
use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
40+
use OCA\DAV\Upload\FutureFile;
4041
use OCP\Files\FileInfo;
4142
use OCP\Files\ForbiddenException;
4243
use OCP\Files\InvalidPathException;

apps/dav/lib/Connector/Sabre/Node.php

+8
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,14 @@ public function getInternalFileId() {
246246
return $this->info->getId();
247247
}
248248

249+
public function getInternalPath(): string {
250+
return $this->info->getInternalPath();
251+
}
252+
253+
public function getAbsoluteInternalPath(): string {
254+
return $this->info->getPath();
255+
}
256+
249257
/**
250258
* @param string $user
251259
* @return int

apps/dav/lib/Connector/Sabre/ServerFactory.php

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
use OC\Files\Node\Folder;
3535
use OCA\DAV\AppInfo\PluginManager;
3636
use OCA\DAV\Files\BrowserErrorPagePlugin;
37+
use OCA\DAV\Service\CustomPropertiesService;
3738
use OCP\Files\Mount\IMountManager;
3839
use OCP\IConfig;
3940
use OCP\IDBConnection;
@@ -204,6 +205,7 @@ public function createServer($baseUri,
204205
new \OCA\DAV\DAV\CustomPropertiesBackend(
205206
$objectTree,
206207
$this->databaseConnection,
208+
\OC::$server->get(CustomPropertiesService::class),
207209
$this->userSession->getUser()
208210
)
209211
)

apps/dav/lib/DAV/CustomPropertiesBackend.php

+10-7
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
namespace OCA\DAV\DAV;
2626

2727
use OCA\DAV\Connector\Sabre\Node;
28+
use OCA\DAV\Service\CustomPropertiesService;
2829
use OCP\IDBConnection;
2930
use OCP\IUser;
3031
use Sabre\DAV\PropertyStorage\Backend\BackendInterface;
@@ -75,6 +76,11 @@ class CustomPropertiesBackend implements BackendInterface {
7576
*/
7677
private $connection;
7778

79+
/**
80+
* @var CustomPropertiesService
81+
*/
82+
private $customPropertiesService;
83+
7884
/**
7985
* @var IUser
8086
*/
@@ -95,9 +101,11 @@ class CustomPropertiesBackend implements BackendInterface {
95101
public function __construct(
96102
Tree $tree,
97103
IDBConnection $connection,
104+
CustomPropertiesService $customPropertiesService,
98105
IUser $user) {
99106
$this->tree = $tree;
100107
$this->connection = $connection;
108+
$this->customPropertiesService = $customPropertiesService;
101109
$this->user = $user;
102110
}
103111

@@ -171,13 +179,8 @@ public function propPatch($path, PropPatch $propPatch) {
171179
* @param string $path path of node for which to delete properties
172180
*/
173181
public function delete($path) {
174-
$statement = $this->connection->prepare(
175-
'DELETE FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ?'
176-
);
177-
$statement->execute([$this->user->getUID(), $this->formatPath($path)]);
178-
$statement->closeCursor();
179-
180-
unset($this->userCache[$path]);
182+
$this->customPropertiesService->delete($this->user->getUID(), $path);
183+
unset($this->cache[$path]);
181184
}
182185

183186
/**

apps/dav/lib/Server.php

+4
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,10 @@
6363
use OCA\DAV\Files\BrowserErrorPagePlugin;
6464
use OCA\DAV\Files\LazySearchBackend;
6565
use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin;
66+
use OCA\DAV\Service\CustomPropertiesService;
6667
use OCA\DAV\SystemTag\SystemTagPlugin;
6768
use OCA\DAV\Upload\ChunkingPlugin;
69+
use OCA\DAV\Upload\ChunkingV2Plugin;
6870
use OCP\EventDispatcher\IEventDispatcher;
6971
use OCP\IRequest;
7072
use OCP\SabrePluginEvent;
@@ -203,6 +205,7 @@ public function __construct(IRequest $request, $baseUri) {
203205
));
204206

205207
$this->server->addPlugin(new CopyEtagHeaderPlugin());
208+
$this->server->addPlugin(new ChunkingV2Plugin());
206209
$this->server->addPlugin(new ChunkingPlugin());
207210

208211
// allow setup of additional plugins
@@ -249,6 +252,7 @@ public function __construct(IRequest $request, $baseUri) {
249252
new CustomPropertiesBackend(
250253
$this->server->tree,
251254
\OC::$server->getDatabaseConnection(),
255+
\OC::$server->get(CustomPropertiesService::class),
252256
\OC::$server->getUserSession()->getUser()
253257
)
254258
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<?php
2+
/*
3+
* @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
4+
*
5+
* @author Julius Härtl <jus@bitgrid.net>
6+
*
7+
* @license GNU AGPL version 3 or any later version
8+
*
9+
* This program is free software: you can redistribute it and/or modify
10+
* it under the terms of the GNU Affero General Public License as
11+
* published by the Free Software Foundation, either version 3 of the
12+
* License, or (at your option) any later version.
13+
*
14+
* This program is distributed in the hope that it will be useful,
15+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
* GNU Affero General Public License for more details.
18+
*
19+
* You should have received a copy of the GNU Affero General Public License
20+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
21+
*
22+
*/
23+
24+
declare(strict_types=1);
25+
26+
27+
namespace OCA\DAV\Service;
28+
29+
use OCP\IDBConnection;
30+
31+
class CustomPropertiesService {
32+
33+
/** @var IDBConnection */
34+
private $connection;
35+
36+
public function __construct(IDBConnection $connection) {
37+
$this->connection = $connection;
38+
}
39+
40+
public function delete(string $userId, string $path): void {
41+
$statement = $this->connection->prepare(
42+
'DELETE FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ?'
43+
);
44+
$result = $statement->execute([$userId, $this->formatPath($path)]);
45+
$result->closeCursor();
46+
}
47+
48+
/**
49+
* long paths are hashed to ensure they fit in the database
50+
*
51+
* @param string $path
52+
* @return string
53+
*/
54+
private function formatPath(string $path): string {
55+
if (strlen($path) > 250) {
56+
return sha1($path);
57+
}
58+
return $path;
59+
}
60+
}

0 commit comments

Comments
 (0)