From d80b1328227ff77b9a5be1764d0aec1319a74f36 Mon Sep 17 00:00:00 2001 From: Jannik Stehle Date: Thu, 23 Nov 2023 15:40:44 +0100 Subject: [PATCH 1/2] perf: speed up folder tree creation during upload Speeds up the folder tree creation during upload by running the process asynchronously and by saving up PROPFIND requests on nested folders. --- .../enhancement-upload-folder-tree-creation | 6 ++ packages/web-app-files/src/HandleUpload.ts | 84 ++++++++++--------- .../tests/unit/HandleUpload.spec.ts | 3 +- .../web-client/src/webdav/createFolder.ts | 9 +- 4 files changed, 60 insertions(+), 42 deletions(-) create mode 100644 changelog/unreleased/enhancement-upload-folder-tree-creation diff --git a/changelog/unreleased/enhancement-upload-folder-tree-creation b/changelog/unreleased/enhancement-upload-folder-tree-creation new file mode 100644 index 00000000000..1d8f68e7ac0 --- /dev/null +++ b/changelog/unreleased/enhancement-upload-folder-tree-creation @@ -0,0 +1,6 @@ +Enhancement: Folder tree creation during upload + +The performance of the folder tree creation during upload has been improved. + +https://github.com/owncloud/web/pull/10057 +https://github.com/owncloud/web/issues/9817 diff --git a/packages/web-app-files/src/HandleUpload.ts b/packages/web-app-files/src/HandleUpload.ts index f2d32aec5c3..b25aa93e813 100644 --- a/packages/web-app-files/src/HandleUpload.ts +++ b/packages/web-app-files/src/HandleUpload.ts @@ -1,7 +1,7 @@ import Uppy, { UppyFile } from '@uppy/core' import BasePlugin from '@uppy/core/lib/BasePlugin.js' import filesize from 'filesize' -import { dirname, join } from 'path' +import { basename, dirname, join } from 'path' import * as uuid from 'uuid' import { Language } from 'vue3-gettext' import { Ref, unref } from 'vue' @@ -269,82 +269,88 @@ export class HandleUpload extends BasePlugin { const space = this.space const { id: currentFolderId, path: currentFolderPath } = this.currentFolder - const createdFolders = [] - const failedFolders = [] + const routeName = filesToUpload[0].meta.routeName + const routeDriveAliasAndItem = filesToUpload[0].meta.routeDriveAliasAndItem + const routeShareId = filesToUpload[0].meta.routeShareId - for (const file of filesToUpload) { - const directory = file.meta.relativeFolder + const failedFolders: string[] = [] + const directoryTree: Record = {} + const topLevelIds: Record = {} - if (!directory || createdFolders.includes(directory)) { - continue + for (const file of filesToUpload.filter(({ meta }) => !!meta.relativeFolder)) { + const folders = file.meta.relativeFolder.split('/').filter(Boolean) + let current = directoryTree + if (folders.length <= 1) { + topLevelIds[file.meta.relativeFolder] = file.meta.topLevelFolderId } + for (const folder of folders) { + current[folder] = current[folder] || {} + current = current[folder] + } + } - const folders = directory.split('/') - let createdSubFolders = '' - for (const subFolder of folders) { - if (!subFolder) { - continue - } - - const folderToCreate = `${createdSubFolders}/${subFolder}` - if (createdFolders.includes(folderToCreate)) { - createdSubFolders += `/${subFolder}` - createdFolders.push(createdSubFolders) - continue - } - - if (failedFolders.includes(folderToCreate)) { - // only care about top level folders, no need to go deeper - break - } + const createDirectoryLevel = async (current: Record, path = '') => { + if (path) { + const isRoot = path.split('/').length <= 1 + path = urlJoin(path, { leadingSlash: true }) + const uploadId = !isRoot ? uuid.v4() : topLevelIds[path] + const relativeFolder = dirname(path) === '/' ? '' : dirname(path) - const uploadId = createdSubFolders ? uuid.v4() : file.meta.topLevelFolderId const uppyResource = { id: uuid.v4(), - name: subFolder, + name: basename(path), isFolder: true, type: 'folder', meta: { - // current space & folder spaceId: space.id, spaceName: space.name, driveAlias: space.driveAlias, driveType: space.driveType, currentFolder: currentFolderPath, currentFolderId, - // upload data - relativeFolder: createdSubFolders, + relativeFolder, uploadId, - // route data - routeName: file.meta.routeName, - routeDriveAliasAndItem: file.meta.routeDriveAliasAndItem, - routeShareId: file.meta.routeShareId + routeName, + routeDriveAliasAndItem, + routeShareId } } + if (failedFolders.includes(relativeFolder)) { + // return if top level folder failed to create + return + } + this.uppyService.publish('addedForUpload', [uppyResource]) try { const folder = await webdav.createFolder(space, { - path: join(currentFolderPath, folderToCreate) + path: urlJoin(currentFolderPath, path), + fetchFolder: isRoot }) this.uppyService.publish('uploadSuccess', { ...uppyResource, meta: { ...uppyResource.meta, fileId: folder?.fileId } }) - - createdSubFolders += `/${subFolder}` - createdFolders.push(createdSubFolders) } catch (error) { if (error.statusCode !== 405) { console.error(error) - failedFolders.push(folderToCreate) + failedFolders.push(path) this.uppyService.publish('uploadError', { file: uppyResource, error }) } } } + + const foldersToBeCreated = Object.keys(current) + const promises = [] + for (const folder of foldersToBeCreated) { + promises.push(createDirectoryLevel(current[folder], join(path, folder))) + } + return Promise.allSettled(promises) } + await createDirectoryLevel(directoryTree) + let filesToRemove: string[] = [] if (failedFolders.length) { // remove files of folders that could not be created diff --git a/packages/web-app-files/tests/unit/HandleUpload.spec.ts b/packages/web-app-files/tests/unit/HandleUpload.spec.ts index b0faf91a7d4..64bc634af07 100644 --- a/packages/web-app-files/tests/unit/HandleUpload.spec.ts +++ b/packages/web-app-files/tests/unit/HandleUpload.spec.ts @@ -90,7 +90,8 @@ describe('HandleUpload', () => { ) expect(mocks.opts.clientService.webdav.createFolder).toHaveBeenCalledTimes(1) expect(mocks.opts.clientService.webdav.createFolder).toHaveBeenCalledWith(mocks.opts.space, { - path: relativeFolder + path: relativeFolder, + fetchFolder: true }) expect(result.length).toBe(1) }) diff --git a/packages/web-client/src/webdav/createFolder.ts b/packages/web-client/src/webdav/createFolder.ts index 1753540a292..d36ca9d940d 100644 --- a/packages/web-client/src/webdav/createFolder.ts +++ b/packages/web-client/src/webdav/createFolder.ts @@ -12,11 +12,16 @@ export const CreateFolderFactory = ( { accessToken }: WebDavOptions ) => { return { - async createFolder(space: SpaceResource, { path }: { path?: string }): Promise { + async createFolder( + space: SpaceResource, + { path, fetchFolder = true }: { path?: string; fetchFolder?: boolean } + ): Promise { const headers = buildAuthHeader(unref(accessToken), space) await dav.mkcol(urlJoin(space.webDavPath, path, { leadingSlash: true }), { headers }) - return getFileInfoFactory.getFileInfo(space, { path }) + if (fetchFolder) { + return getFileInfoFactory.getFileInfo(space, { path }) + } } } } From 494ef9a2196a1f7619a93b558f85e8a6b54e3a5d Mon Sep 17 00:00:00 2001 From: Jannik Stehle Date: Fri, 24 Nov 2023 08:10:38 +0100 Subject: [PATCH 2/2] docs: add changelog item for #9552 --- changelog/unreleased/enhancement-upload-preparation-time | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changelog/unreleased/enhancement-upload-preparation-time diff --git a/changelog/unreleased/enhancement-upload-preparation-time b/changelog/unreleased/enhancement-upload-preparation-time new file mode 100644 index 00000000000..5b5b2184c67 --- /dev/null +++ b/changelog/unreleased/enhancement-upload-preparation-time @@ -0,0 +1,6 @@ +Enhancement: Upload preparation time + +The performance of the preparation time before each upload has been improved. + +https://github.com/owncloud/web/pull/9552 +https://github.com/owncloud/web/issues/9817