S3 direct multipart upload optimisation.

The change was needed to avoid doubling of upload time as the original solution does a complete re-read of the S3 object on final MOVE of NextCloud chunked upload. See nextcloud#27034 for details.

Signed-off-by: Bernd.Rederlechner@t-systems.com <bernd.rederlechner@t-systems.com>

apps/dav/composer/composer/autoload_classmap.php

@@ -243,6 +243,7 @@
  'OCA\\DAV\\Search\\EventsSearchProvider' => $baseDir . '/../lib/Search/EventsSearchProvider.php',
  'OCA\\DAV\\Search\\TasksSearchProvider' => $baseDir . '/../lib/Search/TasksSearchProvider.php',
  'OCA\\DAV\\Server' => $baseDir . '/../lib/Server.php',
+ 'OCA\\DAV\\Service\\CustomPropertiesService' => $baseDir . '/../lib/Service/CustomPropertiesService.php',
  'OCA\\DAV\\Settings\\CalDAVSettings' => $baseDir . '/../lib/Settings/CalDAVSettings.php',
  'OCA\\DAV\\Storage\\PublicOwnerWrapper' => $baseDir . '/../lib/Storage/PublicOwnerWrapper.php',
  'OCA\\DAV\\SystemTag\\SystemTagMappingNode' => $baseDir . '/../lib/SystemTag/SystemTagMappingNode.php',
@@ -255,6 +256,7 @@
  'OCA\\DAV\\Traits\\PrincipalProxyTrait' => $baseDir . '/../lib/Traits/PrincipalProxyTrait.php',
  'OCA\\DAV\\Upload\\AssemblyStream' => $baseDir . '/../lib/Upload/AssemblyStream.php',
  'OCA\\DAV\\Upload\\ChunkingPlugin' => $baseDir . '/../lib/Upload/ChunkingPlugin.php',
+ 'OCA\\DAV\\Upload\\ChunkingV2Plugin' => $baseDir . '/../lib/Upload/ChunkingV2Plugin.php',
  'OCA\\DAV\\Upload\\CleanupService' => $baseDir . '/../lib/Upload/CleanupService.php',
  'OCA\\DAV\\Upload\\FutureFile' => $baseDir . '/../lib/Upload/FutureFile.php',
  'OCA\\DAV\\Upload\\RootCollection' => $baseDir . '/../lib/Upload/RootCollection.php',

apps/dav/composer/composer/autoload_static.php

@@ -258,6 +258,7 @@ class ComposerStaticInitDAV
  'OCA\\DAV\\Search\\EventsSearchProvider' => __DIR__ . '/..' . '/../lib/Search/EventsSearchProvider.php',
  'OCA\\DAV\\Search\\TasksSearchProvider' => __DIR__ . '/..' . '/../lib/Search/TasksSearchProvider.php',
  'OCA\\DAV\\Server' => __DIR__ . '/..' . '/../lib/Server.php',
+ 'OCA\\DAV\\Service\\CustomPropertiesService' => __DIR__ . '/..' . '/../lib/Service/CustomPropertiesService.php',
  'OCA\\DAV\\Settings\\CalDAVSettings' => __DIR__ . '/..' . '/../lib/Settings/CalDAVSettings.php',
  'OCA\\DAV\\Storage\\PublicOwnerWrapper' => __DIR__ . '/..' . '/../lib/Storage/PublicOwnerWrapper.php',
  'OCA\\DAV\\SystemTag\\SystemTagMappingNode' => __DIR__ . '/..' . '/../lib/SystemTag/SystemTagMappingNode.php',
@@ -270,6 +271,7 @@ class ComposerStaticInitDAV
  'OCA\\DAV\\Traits\\PrincipalProxyTrait' => __DIR__ . '/..' . '/../lib/Traits/PrincipalProxyTrait.php',
  'OCA\\DAV\\Upload\\AssemblyStream' => __DIR__ . '/..' . '/../lib/Upload/AssemblyStream.php',
  'OCA\\DAV\\Upload\\ChunkingPlugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingPlugin.php',
+ 'OCA\\DAV\\Upload\\ChunkingV2Plugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingV2Plugin.php',
  'OCA\\DAV\\Upload\\CleanupService' => __DIR__ . '/..' . '/../lib/Upload/CleanupService.php',
  'OCA\\DAV\\Upload\\FutureFile' => __DIR__ . '/..' . '/../lib/Upload/FutureFile.php',
  'OCA\\DAV\\Upload\\RootCollection' => __DIR__ . '/..' . '/../lib/Upload/RootCollection.php',

apps/dav/lib/BackgroundJob/UploadCleanup.php

@@ -29,6 +29,7 @@
 namespace OCA\DAV\BackgroundJob;
 
 use OC\User\NoUserException;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
 use OCP\BackgroundJob\TimedJob;
@@ -45,10 +46,14 @@ class UploadCleanup extends TimedJob {
  /** @var IJobList */
  private $jobList;
 
- public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList) {
+ /** @var CustomPropertiesService */
+ private $customPropertiesService;
+
+ public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList, CustomPropertiesService $customPropertiesService) {
  parent::__construct($time);
  $this->rootFolder = $rootFolder;
  $this->jobList = $jobList;
+ $this->customPropertiesService = $customPropertiesService;
 
  // Run once a day
  $this->setInterval(60 * 60 * 24);
@@ -72,6 +77,8 @@ protected function run($argument) {
 
  $files = $uploadFolder->getDirectoryListing();
 
+ $davPath = 'uploads/' . $uid . '/' . $uploadFolder->getName();
+
  // Remove if all files have an mtime of more than a day
  $time = $this->time->getTime() - 60 * 60 * 24;
 
@@ -83,6 +90,7 @@ protected function run($argument) {
  }, $initial);
 
  if ($expire) {
+ $this->customPropertiesService->delete($uid, $davPath);
  $uploadFolder->delete();
  $this->jobList->remove(self::class, $argument);
  }

apps/dav/lib/Connector/Sabre/Directory.php

@@ -38,6 +38,7 @@
 use OCA\DAV\Connector\Sabre\Exception\FileLocked;
 use OCA\DAV\Connector\Sabre\Exception\Forbidden;
 use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
+use OCA\DAV\Upload\FutureFile;
 use OCP\Files\FileInfo;
 use OCP\Files\ForbiddenException;
 use OCP\Files\InvalidPathException;

apps/dav/lib/Connector/Sabre/Node.php

@@ -248,6 +248,14 @@ public function getInternalFileId() {
  return $this->info->getId();
  }
 
+ public function getInternalPath(): string {
+ return $this->info->getInternalPath();
+ }
+
+ public function getAbsoluteInternalPath(): string {
+ return $this->info->getPath();
+ }
+
  /**
  * @param string $user
  * @return int

apps/dav/lib/Connector/Sabre/ServerFactory.php

@@ -35,6 +35,7 @@
 use OC\Files\Node\Folder;
 use OCA\DAV\AppInfo\PluginManager;
 use OCA\DAV\Files\BrowserErrorPagePlugin;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCP\Files\Mount\IMountManager;
 use OCP\IConfig;
 use OCP\IDBConnection;
@@ -204,6 +205,7 @@ public function createServer($baseUri,
  new \OCA\DAV\DAV\CustomPropertiesBackend(
  $objectTree,
  $this->databaseConnection,
+ \OC::$server->get(CustomPropertiesService::class),
  $this->userSession->getUser()
  )
  )

apps/dav/lib/DAV/CustomPropertiesBackend.php

@@ -26,6 +26,7 @@
 namespace OCA\DAV\DAV;
 
 use OCA\DAV\Connector\Sabre\Node;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCP\IDBConnection;
 use OCP\IUser;
 use Sabre\DAV\PropertyStorage\Backend\BackendInterface;
@@ -63,6 +64,11 @@ class CustomPropertiesBackend implements BackendInterface {
  */
  private $connection;
 
+ /**
+ * @var CustomPropertiesService
+ */
+ private $customPropertiesService;
+
  /**
  * @var IUser
  */
@@ -83,9 +89,11 @@ class CustomPropertiesBackend implements BackendInterface {
  public function __construct(
  Tree $tree,
  IDBConnection $connection,
+ CustomPropertiesService $customPropertiesService,
  IUser $user) {
  $this->tree = $tree;
  $this->connection = $connection;
+ $this->customPropertiesService = $customPropertiesService;
  $this->user = $user;
  }
 
@@ -155,12 +163,7 @@ public function propPatch($path, PropPatch $propPatch) {
  * @param string $path path of node for which to delete properties
  */
  public function delete($path) {
- $statement = $this->connection->prepare(
- 'DELETE FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ?'
- );
- $statement->execute([$this->user->getUID(), $this->formatPath($path)]);
- $statement->closeCursor();
-
+ $this->customPropertiesService->delete($this->user->getUID(), $path);
  unset($this->cache[$path]);
  }
 

apps/dav/lib/Server.php

@@ -64,8 +64,10 @@
 use OCA\DAV\Files\BrowserErrorPagePlugin;
 use OCA\DAV\Files\LazySearchBackend;
 use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCA\DAV\SystemTag\SystemTagPlugin;
 use OCA\DAV\Upload\ChunkingPlugin;
+use OCA\DAV\Upload\ChunkingV2Plugin;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\IRequest;
 use OCP\SabrePluginEvent;
@@ -203,6 +205,7 @@ public function __construct(IRequest $request, $baseUri) {
  ));
 
  $this->server->addPlugin(new CopyEtagHeaderPlugin());
+ $this->server->addPlugin(new ChunkingV2Plugin());
  $this->server->addPlugin(new ChunkingPlugin());
 
  // allow setup of additional plugins
@@ -248,6 +251,7 @@ public function __construct(IRequest $request, $baseUri) {
  new CustomPropertiesBackend(
  $this->server->tree,
  \OC::$server->getDatabaseConnection(),
+ \OC::$server->get(CustomPropertiesService::class),
  \OC::$server->getUserSession()->getUser()
  )
  )

apps/dav/lib/Service/CustomPropertiesService.php

@@ -0,0 +1,60 @@
+<?php
+/*
+ * @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+declare(strict_types=1);
+
+
+namespace OCA\DAV\Service;
+
+use OCP\IDBConnection;
+
+class CustomPropertiesService {
+
+ /** @var IDBConnection */
+ private $connection;
+
+ public function __construct(IDBConnection $connection) {
+ $this->connection = $connection;
+ }
+
+ public function delete(string $userId, string $path): void {
+ $statement = $this->connection->prepare(
+ 'DELETE FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ?'
+ );
+ $result = $statement->execute([$userId, $this->formatPath($path)]);
+ $result->closeCursor();
+ }
+
+ /**
+ * long paths are hashed to ensure they fit in the database
+ *
+ * @param string $path
+ * @return string
+ */
+ private function formatPath(string $path): string {
+ if (strlen($path) > 250) {
+ return sha1($path);
+ }
+ return $path;
+ }
+}