improve download, allow download of big files, works on ff and chrome

CHANGELOG.md

@@ -64,6 +64,7 @@ Breaking changes:
 - [preferences] renamed overridenPreferenceName to overriddenPreferenceName
 - [task] `cwd`, which used to be defined directly under `Task`, is moved into `Task.options` object
 - [workspace] `isMultiRootWorkspaceOpened()` is renamed into `isMultiRootWorkspaceEnabled()`
+- [filesystem] Changed `FileDownloadService` API to support streaming download of huge files.
 
 ## v0.7.0
 

packages/filesystem/package.json

@@ -6,7 +6,6 @@
     "@theia/core": "^0.8.0",
     "@types/body-parser": "^1.17.0",
     "@types/fs-extra": "^4.0.2",
-    "@types/mime-types": "^2.1.0",
     "@types/rimraf": "^2.0.2",
     "@types/tar-fs": "^1.16.1",
     "@types/touch": "0.0.1",
@@ -15,7 +14,6 @@
     "drivelist": "^6.4.3",
     "fs-extra": "^4.0.2",
     "http-status-codes": "^1.3.0",
-    "mime-types": "^2.1.18",
     "minimatch": "^3.0.4",
     "mv": "^2.1.1",
     "rimraf": "^2.6.2",

packages/filesystem/src/browser/download/file-download-command-contribution.ts

@@ -16,10 +16,11 @@
 
 import { inject, injectable } from 'inversify';
 import URI from '@theia/core/lib/common/uri';
+import { isChrome } from '@theia/core/lib/browser/browser';
 import { environment } from '@theia/application-package/lib/environment';
 import { SelectionService } from '@theia/core/lib/common/selection-service';
 import { Command, CommandContribution, CommandRegistry } from '@theia/core/lib/common/command';
-import { UriAwareCommandHandler, UriCommandHandler } from '@theia/core/lib/common/uri-command-handler';
+import { UriAwareCommandHandler } from '@theia/core/lib/common/uri-command-handler';
 import { FileDownloadService } from './file-download-service';
 
 @injectable()
@@ -32,20 +33,26 @@ export class FileDownloadCommandContribution implements CommandContribution {
     protected readonly selectionService: SelectionService;
 
     registerCommands(registry: CommandRegistry): void {
-        const handler = new UriAwareCommandHandler<URI[]>(this.selectionService, this.downloadHandler(), { multi: true });
-        registry.registerCommand(FileDownloadCommands.DOWNLOAD, handler);
+        registry.registerCommand(
+            FileDownloadCommands.DOWNLOAD,
+            new UriAwareCommandHandler<URI[]>(this.selectionService, {
+                execute: uris => this.executeDownload(uris),
+                isEnabled: uris => this.isDownloadEnabled(uris),
+                isVisible: uris => this.isDownloadVisible(uris),
+            }, { multi: true })
+        );
+        registry.registerCommand(
+            FileDownloadCommands.COPY_DOWNLOAD_LINK,
+            new UriAwareCommandHandler<URI[]>(this.selectionService, {
+                execute: uris => this.executeDownload(uris, { copyLink: true }),
+                isEnabled: uris => isChrome && this.isDownloadEnabled(uris),
+                isVisible: uris => isChrome && this.isDownloadVisible(uris),
+            }, { multi: true })
+        );
     }
 
-    protected downloadHandler(): UriCommandHandler<URI[]> {
-        return {
-            execute: uris => this.executeDownload(uris),
-            isEnabled: uris => this.isDownloadEnabled(uris),
-            isVisible: uris => this.isDownloadVisible(uris),
-        };
-    }
-
-    protected async executeDownload(uris: URI[]): Promise<void> {
-        this.downloadService.download(uris);
+    protected async executeDownload(uris: URI[], options?: { copyLink?: boolean }): Promise<void> {
+        this.downloadService.download(uris, options);
     }
 
     protected isDownloadEnabled(uris: URI[]): boolean {
@@ -66,4 +73,10 @@ export namespace FileDownloadCommands {
         label: 'Download'
     };
 
+    export const COPY_DOWNLOAD_LINK: Command = {
+        id: 'file.copyDownloadLink',
+        category: 'File',
+        label: 'Copy Download Link'
+    };
+
 }

packages/filesystem/src/browser/download/file-download-service.ts

@@ -18,18 +18,15 @@ import { inject, injectable } from 'inversify';
 import URI from '@theia/core/lib/common/uri';
 import { ILogger } from '@theia/core/lib/common/logger';
 import { Endpoint } from '@theia/core/lib/browser/endpoint';
-import { StatusBar, StatusBarAlignment } from '@theia/core/lib/browser/status-bar';
 import { FileSystem } from '../../common/filesystem';
 import { FileDownloadData } from '../../common/download/file-download-data';
 import { MessageService } from '@theia/core/lib/common/message-service';
+import { addClipboardListener } from '@theia/core/lib/browser/widgets';
 
 @injectable()
 export class FileDownloadService {
 
-    protected static PREPARING_DOWNLOAD_ID = 'theia-preparing-download';
-
     protected anchor: HTMLAnchorElement | undefined;
-    protected downloadQueue: number[] = [];
     protected downloadCounter: number = 0;
 
     @inject(ILogger)
@@ -38,60 +35,72 @@ export class FileDownloadService {
     @inject(FileSystem)
     protected readonly fileSystem: FileSystem;
 
-    @inject(StatusBar)
-    protected readonly statusBar: StatusBar;
-
     @inject(MessageService)
     protected readonly messageService: MessageService;
 
-    async download(uris: URI[]): Promise<void> {
+    protected handleCopy(event: ClipboardEvent, downloadUrl: string): void {
+        if (downloadUrl && event.clipboardData) {
+            event.clipboardData.setData('text/plain', downloadUrl);
+            event.preventDefault();
+            this.messageService.info('Copied the download link to the clipboard.');
+        }
+    }
+
+    async cancelDownload(id: string): Promise<void> {
+        await fetch(`${this.endpoint()}/download/?id=${id}&cancel=true`);
+    }
+
+    async download(uris: URI[], options?: FileDownloadService.DownloadOptions): Promise<void> {
+        let cancel = false;
         if (uris.length === 0) {
             return;
         }
-        let downloadId: number | undefined;
+        const copyLink = options && options.copyLink ? true : false;
         try {
-            downloadId = this.downloadCounter++;
-            if (this.downloadQueue.length === 0) {
-                await this.statusBar.setElement(FileDownloadService.PREPARING_DOWNLOAD_ID, {
-                    alignment: StatusBarAlignment.RIGHT,
-                    text: '$(spinner~spin) Preparing download...',
-                    tooltip: 'Preparing download...',
-                    priority: 1
-                });
+            const [progress, result] = await Promise.all([
+                this.messageService.showProgress({
+                    text: `Preparing download${copyLink ? ' link' : ''}...`, options: { cancelable: true }
+                }, () => { cancel = true; }),
+                // tslint:disable-next-line:no-any
+                new Promise<{ response: Response, jsonResponse: any }>(async resolve => {
+                    const resp = await fetch(this.request(uris));
+                    const jsonResp = await resp.json();
+                    resolve({ response: resp, jsonResponse: jsonResp });
+                })
+            ]);
+            const { response, jsonResponse } = result;
+            if (cancel) {
+                this.cancelDownload(jsonResponse.id);
+                return;
             }
-            this.downloadQueue.push(downloadId);
-            const response = await fetch(this.request(uris));
-            await this.statusBar.removeElement(FileDownloadService.PREPARING_DOWNLOAD_ID);
-            const title = await this.title(response, uris);
             const { status, statusText } = response;
             if (status === 200) {
-                await this.forceDownload(response, decodeURIComponent(title));
+                progress.cancel();
+                const downloadUrl = `${this.endpoint()}/download/?id=${jsonResponse.id}`;
+                if (copyLink) {
+                    if (document.documentElement) {
+                        addClipboardListener(document.documentElement, 'copy', e => this.handleCopy(e, downloadUrl));
+                        document.execCommand('copy');
+                    }
+                } else {
+                    this.forceDownload(jsonResponse.id, decodeURIComponent(jsonResponse.name));
+                }
             } else {
                 throw new Error(`Received unexpected status code: ${status}. [${statusText}]`);
             }
         } catch (e) {
             this.logger.error(`Error occurred when downloading: ${uris.map(u => u.toString(true))}.`, e);
-        } finally {
-            if (downloadId !== undefined) {
-                const indexOf = this.downloadQueue.indexOf(downloadId);
-                if (indexOf !== -1) {
-                    this.downloadQueue.splice(indexOf, 1);
-                }
-                if (this.downloadQueue.length === 0) {
-                    this.statusBar.removeElement(FileDownloadService.PREPARING_DOWNLOAD_ID);
-                }
-            }
         }
     }
 
-    protected async forceDownload(response: Response, title: string): Promise<void> {
+    protected async forceDownload(id: string, title: string): Promise<void> {
         let url: string | undefined;
         try {
-            const blob = await response.blob();
-            url = URL.createObjectURL(blob);
             if (this.anchor === undefined) {
                 this.anchor = document.createElement('a');
             }
+            const endpoint = this.endpoint();
+            url = `${endpoint}/download/?id=${id}`;
             this.anchor.href = url;
             this.anchor.style.display = 'none';
             this.anchor.download = title;
@@ -108,24 +117,6 @@ export class FileDownloadService {
         }
     }
 
-    protected async title(response: Response, uris: URI[]): Promise<string> {
-        let title = (response.headers.get('Content-Disposition') || '').split('attachment; filename=').pop();
-        if (title) {
-            return title;
-        }
-        // tslint:disable-next-line:whitespace
-        const [uri,] = uris;
-        if (uris.length === 1) {
-            const stat = await this.fileSystem.getFileStat(uri.toString());
-            if (stat === undefined) {
-                throw new Error(`Unexpected error occurred when downloading file. Files does not exist. URI: ${uri.toString(true)}.`);
-            }
-            title = uri.path.base;
-            return stat.isDirectory ? `${title}.tar` : title;
-        }
-        return `${uri.parent.path.name}.tar`;
-    }
-
     protected request(uris: URI[]): Request {
         const url = this.url(uris);
         const init = this.requestInit(uris);
@@ -173,3 +164,12 @@ export class FileDownloadService {
     }
 
 }
+
+export namespace FileDownloadService {
+    export interface DownloadOptions {
+        /**
+         * `true` if the download link has to be copied to the clipboard. This will not trigger the actual download. Defaults to `false`.
+         */
+        readonly copyLink?: boolean;
+    }
+}

packages/filesystem/src/node/download/file-download-backend-module.ts

@@ -17,13 +17,16 @@
 import { ContainerModule } from 'inversify';
 import { BackendApplicationContribution } from '@theia/core/lib/node/backend-application';
 import { FileDownloadEndpoint } from './file-download-endpoint';
-import { FileDownloadHandler, SingleFileDownloadHandler, MultiFileDownloadHandler } from './file-download-handler';
+import { FileDownloadHandler, SingleFileDownloadHandler, MultiFileDownloadHandler, DownloadLinkHandler } from './file-download-handler';
 import { DirectoryArchiver } from './directory-archiver';
+import { FileDownloadCache } from './file-download-cache';
 
 export default new ContainerModule(bind => {
     bind(FileDownloadEndpoint).toSelf().inSingletonScope();
     bind(BackendApplicationContribution).toService(FileDownloadEndpoint);
+    bind(FileDownloadCache).toSelf().inSingletonScope();
     bind(FileDownloadHandler).to(SingleFileDownloadHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.SINGLE);
     bind(FileDownloadHandler).to(MultiFileDownloadHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.MULTI);
+    bind(FileDownloadHandler).to(DownloadLinkHandler).inSingletonScope().whenTargetNamed(FileDownloadHandler.DOWNLOAD_LINK);
     bind(DirectoryArchiver).toSelf().inSingletonScope();
 });

packages/filesystem/src/node/download/file-download-cache.ts

@@ -0,0 +1,88 @@
+/********************************************************************************
+ * Copyright (C) 2019 Bitsler and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+import { injectable, inject } from 'inversify';
+import { ILogger } from '@theia/core/lib/common/logger';
+import * as rimraf from 'rimraf';
+
+export interface DownloadStorageItem {
+    file: string;
+    root?: string;
+    size: number;
+    remove: boolean;
+    expire?: number;
+}
+
+@injectable()
+export class FileDownloadCache {
+
+    @inject(ILogger)
+    protected readonly logger: ILogger;
+    protected readonly downloads = new Map<string, DownloadStorageItem>();
+    protected readonly expireTimeInMinutes: number = 1;
+
+    addDownload(id: string, downloadInfo: DownloadStorageItem): void {
+        downloadInfo.file = encodeURIComponent(downloadInfo.file);
+        if (downloadInfo.root) {
+            downloadInfo.root = encodeURIComponent(downloadInfo.root);
+        }
+        // expires in 1 minute enough for parallel connections to be connected.
+        downloadInfo.expire = Date.now() + (this.expireTimeInMinutes * 600000);
+        this.downloads.set(id, downloadInfo);
+    }
+
+    getDownload(id: string): DownloadStorageItem | undefined {
+        this.expireDownloads();
+        const downloadInfo = this.downloads.get(id);
+        if (downloadInfo) {
+
+            downloadInfo.file = decodeURIComponent(downloadInfo.file);
+            if (downloadInfo.root) {
+                downloadInfo.root = decodeURIComponent(downloadInfo.root);
+            }
+        }
+        return downloadInfo;
+    }
+
+    deleteDownload(id: string): void {
+        const downloadInfo = this.downloads.get(id);
+        if (downloadInfo && downloadInfo.remove) {
+            this.deleteRecursively(downloadInfo.root || downloadInfo.file);
+        }
+        this.downloads.delete(id);
+    }
+
+    values(): { [key: string]: DownloadStorageItem } {
+        this.expireDownloads();
+        return [...this.downloads.entries()].reduce((downloads, [key, value]) => ({ ...downloads, [key]: value }), {});
+    }
+
+    protected deleteRecursively(pathToDelete: string): void {
+        rimraf(pathToDelete, error => {
+            if (error) {
+                this.logger.warn(`An error occurred while deleting the temporary data from the disk. Cannot clean up: ${pathToDelete}.`, error);
+            }
+        });
+    }
+
+    protected expireDownloads(): void {
+        const time = Date.now();
+        for (const [id, download] of this.downloads.entries()) {
+            if (download.expire && download.expire <= time) {
+                this.deleteDownload(id);
+            }
+        }
+    }
+}

packages/filesystem/src/node/download/file-download-endpoint.ts

@@ -36,8 +36,13 @@ export class FileDownloadEndpoint implements BackendApplicationContribution {
     @named(FileDownloadHandler.MULTI)
     protected readonly multiFileDownloadHandler: FileDownloadHandler;
 
+    @inject(FileDownloadHandler)
+    @named(FileDownloadHandler.DOWNLOAD_LINK)
+    protected readonly downloadLinkHandler: FileDownloadHandler;
+
     configure(app: Application): void {
         const router = Router();
+        router.get('/download', (request, response) => this.downloadLinkHandler.handle(request, response));
         router.get('/', (request, response) => this.singleFileDownloadHandler.handle(request, response));
         router.put('/', (request, response) => this.multiFileDownloadHandler.handle(request, response));
         // Content-Type: application/json