Merge #247 [backport25] S3 Multipart turbo upload

Author: memurats

apps/dav/composer/composer/autoload_classmap.php

@@ -297,6 +297,7 @@
     'OCA\\DAV\\Search\\EventsSearchProvider' => $baseDir . '/../lib/Search/EventsSearchProvider.php',
     'OCA\\DAV\\Search\\TasksSearchProvider' => $baseDir . '/../lib/Search/TasksSearchProvider.php',
     'OCA\\DAV\\Server' => $baseDir . '/../lib/Server.php',
+    'OCA\\DAV\\Service\\CustomPropertiesService' => $baseDir . '/../lib/Service/CustomPropertiesService.php',
     'OCA\\DAV\\Settings\\AvailabilitySettings' => $baseDir . '/../lib/Settings/AvailabilitySettings.php',
     'OCA\\DAV\\Settings\\CalDAVSettings' => $baseDir . '/../lib/Settings/CalDAVSettings.php',
     'OCA\\DAV\\Storage\\PublicOwnerWrapper' => $baseDir . '/../lib/Storage/PublicOwnerWrapper.php',
@@ -311,8 +312,10 @@
     'OCA\\DAV\\Traits\\PrincipalProxyTrait' => $baseDir . '/../lib/Traits/PrincipalProxyTrait.php',
     'OCA\\DAV\\Upload\\AssemblyStream' => $baseDir . '/../lib/Upload/AssemblyStream.php',
     'OCA\\DAV\\Upload\\ChunkingPlugin' => $baseDir . '/../lib/Upload/ChunkingPlugin.php',
+    'OCA\\DAV\\Upload\\ChunkingV2Plugin' => $baseDir . '/../lib/Upload/ChunkingV2Plugin.php',
     'OCA\\DAV\\Upload\\CleanupService' => $baseDir . '/../lib/Upload/CleanupService.php',
     'OCA\\DAV\\Upload\\FutureFile' => $baseDir . '/../lib/Upload/FutureFile.php',
+    'OCA\\DAV\\Upload\\PartFile' => $baseDir . '/../lib/Upload/PartFile.php',
     'OCA\\DAV\\Upload\\RootCollection' => $baseDir . '/../lib/Upload/RootCollection.php',
     'OCA\\DAV\\Upload\\UploadFile' => $baseDir . '/../lib/Upload/UploadFile.php',
     'OCA\\DAV\\Upload\\UploadFolder' => $baseDir . '/../lib/Upload/UploadFolder.php',

apps/dav/composer/composer/autoload_static.php

@@ -312,6 +312,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Search\\EventsSearchProvider' => __DIR__ . '/..' . '/../lib/Search/EventsSearchProvider.php',
         'OCA\\DAV\\Search\\TasksSearchProvider' => __DIR__ . '/..' . '/../lib/Search/TasksSearchProvider.php',
         'OCA\\DAV\\Server' => __DIR__ . '/..' . '/../lib/Server.php',
+        'OCA\\DAV\\Service\\CustomPropertiesService' => __DIR__ . '/..' . '/../lib/Service/CustomPropertiesService.php',
         'OCA\\DAV\\Settings\\AvailabilitySettings' => __DIR__ . '/..' . '/../lib/Settings/AvailabilitySettings.php',
         'OCA\\DAV\\Settings\\CalDAVSettings' => __DIR__ . '/..' . '/../lib/Settings/CalDAVSettings.php',
         'OCA\\DAV\\Storage\\PublicOwnerWrapper' => __DIR__ . '/..' . '/../lib/Storage/PublicOwnerWrapper.php',
@@ -326,8 +327,10 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Traits\\PrincipalProxyTrait' => __DIR__ . '/..' . '/../lib/Traits/PrincipalProxyTrait.php',
         'OCA\\DAV\\Upload\\AssemblyStream' => __DIR__ . '/..' . '/../lib/Upload/AssemblyStream.php',
         'OCA\\DAV\\Upload\\ChunkingPlugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingPlugin.php',
+        'OCA\\DAV\\Upload\\ChunkingV2Plugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingV2Plugin.php',
         'OCA\\DAV\\Upload\\CleanupService' => __DIR__ . '/..' . '/../lib/Upload/CleanupService.php',
         'OCA\\DAV\\Upload\\FutureFile' => __DIR__ . '/..' . '/../lib/Upload/FutureFile.php',
+        'OCA\\DAV\\Upload\\PartFile' => __DIR__ . '/..' . '/../lib/Upload/PartFile.php',
         'OCA\\DAV\\Upload\\RootCollection' => __DIR__ . '/..' . '/../lib/Upload/RootCollection.php',
         'OCA\\DAV\\Upload\\UploadFile' => __DIR__ . '/..' . '/../lib/Upload/UploadFile.php',
         'OCA\\DAV\\Upload\\UploadFolder' => __DIR__ . '/..' . '/../lib/Upload/UploadFolder.php',

apps/dav/lib/BackgroundJob/UploadCleanup.php

@@ -28,6 +28,7 @@
 namespace OCA\DAV\BackgroundJob;
 
 use OC\User\NoUserException;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJob;
 use OCP\BackgroundJob\IJobList;
@@ -42,12 +43,14 @@
 class UploadCleanup extends TimedJob {
 	private IRootFolder $rootFolder;
 	private IJobList $jobList;
+	private CustomPropertiesService $customPropertiesService;
 	private LoggerInterface $logger;
 
-	public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList, LoggerInterface $logger) {
+	public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList, CustomPropertiesService $customPropertiesService, LoggerInterface $logger) {
 		parent::__construct($time);
 		$this->rootFolder = $rootFolder;
 		$this->jobList = $jobList;
+		$this->customPropertiesService = $customPropertiesService;
 		$this->logger = $logger;
 
 		// Run once a day
@@ -70,6 +73,10 @@ protected function run($argument) {
 			return;
 		}
 
+		$files = $uploadFolder->getDirectoryListing();
+
+		$davPath = 'uploads/' . $uid . '/' . $uploadFolder->getName();
+
 		// Remove if all files have an mtime of more than a day
 		$time = $this->time->getTime() - 60 * 60 * 24;
 
@@ -93,6 +100,7 @@ protected function run($argument) {
 		}, $initial);
 
 		if ($expire) {
+			$this->customPropertiesService->delete($uid, $davPath);
 			$uploadFolder->delete();
 			$this->jobList->remove(self::class, $argument);
 		}

apps/dav/lib/Capabilities.php

@@ -36,7 +36,7 @@ public function __construct(IConfig $config) {
 	public function getCapabilities() {
 		$capabilities = [
 			'dav' => [
-				'chunking' => '1.0',
+				'chunking' => '2.0'
 			]
 		];
 		if ($this->config->getSystemValueBool('bulkupload.enabled', true)) {

apps/dav/lib/Connector/Sabre/Directory.php

@@ -39,6 +39,7 @@
 use OCA\DAV\Connector\Sabre\Exception\FileLocked;
 use OCA\DAV\Connector\Sabre\Exception\Forbidden;
 use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
+use OCA\DAV\Upload\FutureFile;
 use OCP\Files\FileInfo;
 use OCP\Files\Folder;
 use OCP\Files\ForbiddenException;

apps/dav/lib/Connector/Sabre/Node.php

@@ -269,6 +269,14 @@ public function getInternalFileId() {
 		return $this->info->getId();
 	}
 
+	public function getInternalPath(): string {
+		return $this->info->getInternalPath();
+	}
+
+	public function getAbsoluteInternalPath(): string {
+		return $this->info->getPath();
+	}
+
 	/**
 	 * @param string $user
 	 * @return int

apps/dav/lib/Connector/Sabre/ServerFactory.php

@@ -35,6 +35,7 @@
 use OCA\DAV\AppInfo\PluginManager;
 use OCA\DAV\DAV\ViewOnlyPlugin;
 use OCA\DAV\Files\BrowserErrorPagePlugin;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCP\Files\Mount\IMountManager;
 use OCP\IConfig;
 use OCP\IDBConnection;
@@ -190,6 +191,7 @@ public function createServer(string $baseUri,
 						new \OCA\DAV\DAV\CustomPropertiesBackend(
 							$objectTree,
 							$this->databaseConnection,
+							\OC::$server->get(CustomPropertiesService::class),
 							$this->userSession->getUser()
 						)
 					)

apps/dav/lib/DAV/CustomPropertiesBackend.php

@@ -28,6 +28,7 @@
 use Exception;
 use OCA\DAV\Connector\Sabre\Directory;
 use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCP\IDBConnection;
 use OCP\IUser;
 use Sabre\DAV\PropertyStorage\Backend\BackendInterface;
@@ -116,6 +117,11 @@ class CustomPropertiesBackend implements BackendInterface {
 	 */
 	private $connection;
 
+	/**
+	 * @var CustomPropertiesService
+	 */
+	private $customPropertiesService;
+
 	/**
 	 * @var IUser
 	 */
@@ -136,10 +142,12 @@ class CustomPropertiesBackend implements BackendInterface {
 	public function __construct(
 		Tree $tree,
 		IDBConnection $connection,
+        CustomPropertiesService $customPropertiesService,
 		IUser $user
 	) {
 		$this->tree = $tree;
 		$this->connection = $connection;
+		$this->customPropertiesService = $customPropertiesService;
 		$this->user = $user;
 	}
 
@@ -218,12 +226,7 @@ public function propPatch($path, PropPatch $propPatch) {
 	 * @param string $path path of node for which to delete properties
 	 */
 	public function delete($path) {
-		$statement = $this->connection->prepare(
-			'DELETE FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ?'
-		);
-		$statement->execute([$this->user->getUID(), $this->formatPath($path)]);
-		$statement->closeCursor();
-
+		$this->customPropertiesService->delete($this->user->getUID(), $path);
 		unset($this->userCache[$path]);
 	}
 

apps/dav/lib/Server.php

@@ -68,10 +68,12 @@
 use OCA\DAV\Files\LazySearchBackend;
 use OCA\DAV\Profiler\ProfilerPlugin;
 use OCA\DAV\Provisioning\Apple\AppleProvisioningPlugin;
+use OCA\DAV\Service\CustomPropertiesService;
 use OCA\DAV\SystemTag\SystemTagPlugin;
 use OCA\DAV\Upload\ChunkingPlugin;
 use OCP\AppFramework\Http\Response;
 use OCP\Diagnostics\IEventLogger;
+use OCA\DAV\Upload\ChunkingV2Plugin;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\IRequest;
 use OCP\Profiler\IProfiler;
@@ -215,6 +217,7 @@ public function __construct(IRequest $request, string $baseUri) {
 
 		$this->server->addPlugin(new CopyEtagHeaderPlugin());
 		$this->server->addPlugin(new RequestIdHeaderPlugin(\OC::$server->get(IRequest::class)));
+		$this->server->addPlugin(new ChunkingV2Plugin());
 		$this->server->addPlugin(new ChunkingPlugin());
 
 		// allow setup of additional plugins
@@ -267,6 +270,7 @@ public function __construct(IRequest $request, string $baseUri) {
 						new CustomPropertiesBackend(
 							$this->server->tree,
 							\OC::$server->getDatabaseConnection(),
+							\OC::$server->get(CustomPropertiesService::class),
 							\OC::$server->getUserSession()->getUser()
 						)
 					)

apps/dav/lib/Service/CustomPropertiesService.php

@@ -0,0 +1,60 @@
+<?php
+/*
+ * @copyright Copyright (c) 2021 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+declare(strict_types=1);
+
+
+namespace OCA\DAV\Service;
+
+use OCP\IDBConnection;
+
+class CustomPropertiesService {
+
+	/** @var IDBConnection */
+	private $connection;
+
+	public function __construct(IDBConnection $connection) {
+		$this->connection = $connection;
+	}
+
+	public function delete(string $userId, string $path): void {
+		$statement = $this->connection->prepare(
+			'DELETE FROM `*PREFIX*properties` WHERE `userid` = ? AND `propertypath` = ?'
+		);
+		$result = $statement->execute([$userId, $this->formatPath($path)]);
+		$result->closeCursor();
+	}
+
+	/**
+	 * long paths are hashed to ensure they fit in the database
+	 *
+	 * @param string $path
+	 * @return string
+	 */
+	private function formatPath(string $path): string {
+		if (strlen($path) > 250) {
+			return sha1($path);
+		}
+		return $path;
+	}
+}