From 3f7a60a06db061286044353f18b38b8140f7f2b7 Mon Sep 17 00:00:00 2001 From: John Vilk Date: Tue, 1 Nov 2016 22:57:44 -0400 Subject: [PATCH] Faster builds, HTML5FS optimization, support ZipFS decompression plugins. --- package.json | 39 ++++++++-- src/backend/HTML5FS.ts | 165 ++++++++++++++++++++--------------------- src/backend/ZipFS.ts | 49 ++++++++---- 3 files changed, 147 insertions(+), 106 deletions(-) diff --git a/package.json b/package.json index 85afa414..8155f6da 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "karma-opera-launcher": "^1.0.0", "karma-safari-launcher": "^1.0.0", "mocha": "^3.1.1", + "npm-run-all": "^3.1.1", "object-wrapper": "^0.2.0", "remap-istanbul": "^0.6.4", "rimraf": "^2.5.4", @@ -68,11 +69,39 @@ }, "scripts": { "lint": "tslint -c src/tslint.json --project src/tsconfig.json", - "build": "tsc -p src && rollup -c src/rollup.config.js && webpack --config src/webpack.config.js && webpack -p --config src/webpack.config.js", - "build_tests": "tsc -p test && rollup -c test/rollup.config.js && rollup -c test/rollup.worker.config.js && webpack --config test/webpack.config.js", - "build_scripts": "tsc -p scripts", - "dist": "npm run build && npm run build_scripts && npm run lint && node build/scripts/make_dist.js && tsc -p src/tsconfig.node.json", - "test": "npm run build_scripts && node build/scripts/make_fixture_loader.js && node build/scripts/make_test_launcher.js && npm run build_tests && node build/scripts/make_zip_fixtures.js && node build/scripts/make_xhrfs_index.js test/fixtures/xhrfs/listings.json && karma start karma.config.js", + "build:tsc": "tsc -p src", + "watch:tsc": "tsc -p src --watch", + "build:scripts": "tsc -p scripts", + "watch:scripts": "tsc -p scripts --watch", + "build:rollup": "rollup -c src/rollup.config.js", + "watch:rollup": "rollup -w -c src/rollup.config.js", + "build:webpack": "webpack --config src/webpack.config.js", + "watch:webpack": "webpack -w --config src/webpack.config.js", + "build:webpack-release": "webpack -p --config src/webpack.config.js", + "watch:webpack-release": "webpack -w -p --config src/webpack.config.js", + "build": "npm-run-all --parallel build:tsc build:scripts --sequential build:rollup --parallel build:webpack build:webpack-release", + "watch": "npm-run-all build --parallel watch:tsc watch:scripts watch:rollup watch:webpack watch:webpack-release", + "test:build:tsc": "tsc -p test", + "test:watch:tsc": "tsc --watch -p test", + "test:build:rollup": "rollup -c test/rollup.config.js", + "test:watch:rollup": "rollup -w -c test/rollup.config.js", + "test:build:rollup-worker": "rollup -c test/rollup.worker.config.js", + "test:watch:rollup-worker": "rollup -w -c test/rollup.worker.config.js", + "test:build:webpack": "webpack --config test/webpack.config.js", + "test:watch:webpack": "webpack -w --config test/webpack.config.js", + "test:build": "npm-run-all test:build:tsc --parallel test:build:rollup test:build:rollup-worker --sequential test:build:webpack", + "test:watch": "npm-run-all --parallel test:watch:tsc test:watch:rollup test:watch:rollup-worker test:watch:webpack", + "dist:build:node": "tsc -p src/tsconfig.node.json", + "script:make_dist": "node build/scripts/make_dist.js", + "dist": "npm-run-all build lint script:make_dist dist:build:node", + "script:make_fixture_loader": "node build/scripts/make_fixture_loader.js", + "script:make_test_launcher": "node build/scripts/make_test_launcher.js", + "script:make_zip_fixtures": "node build/scripts/make_zip_fixtures", + "script:make_xhrfs_index": "node build/scripts/make_xhrfs_index.js test/fixtures/xhrfs/listings.json", + "test:karma": "karma start karma.config.js", + "test:prepare": "npm-run-all build:scripts script:make_fixture_loader script:make_test_launcher test:build script:make_zip_fixtures script:make_xhrfs_index", + "test": "npm-run-all test:prepare test:karma", + "watch-test": "npm-run-all test:prepare --parallel watch:scripts test:watch test:karma", "prepublish": "npm run dist" }, "dependencies": { diff --git a/src/backend/HTML5FS.ts b/src/backend/HTML5FS.ts index 1d2c3741..a77c0c39 100644 --- a/src/backend/HTML5FS.ts +++ b/src/backend/HTML5FS.ts @@ -42,6 +42,50 @@ function _toArray(list?: any[]): any[] { return Array.prototype.slice.call(list || [], 0); } +/** + * Converts the given DOMError into an appropriate ApiError. + * Full list of values here: + * https://developer.mozilla.org/en-US/docs/Web/API/DOMError + */ +function convertError(err: DOMError, p: string, expectedDir: boolean): ApiError { + switch (err.name) { + /* The user agent failed to create a file or directory due to the existence of a file or + directory with the same path. */ + case "PathExistsError": + return ApiError.EEXIST(p); + /* The operation failed because it would cause the application to exceed its storage quota. */ + case 'QuotaExceededError': + return ApiError.FileError(ErrorCode.ENOSPC, p); + /* A required file or directory could not be found at the time an operation was processed. */ + case 'NotFoundError': + return ApiError.ENOENT(p); + /* This is a security error code to be used in situations not covered by any other error codes. + - A required file was unsafe for access within a Web application + - Too many calls are being made on filesystem resources */ + case 'SecurityError': + return ApiError.FileError(ErrorCode.EACCES, p); + /* The modification requested was illegal. Examples of invalid modifications include moving a + directory into its own child, moving a file into its parent directory without changing its name, + or copying a directory to a path occupied by a file. */ + case 'InvalidModificationError': + return ApiError.FileError(ErrorCode.EPERM, p); + /* The user has attempted to look up a file or directory, but the Entry found is of the wrong type + [e.g. is a DirectoryEntry when the user requested a FileEntry]. */ + case 'TypeMismatchError': + return ApiError.FileError(expectedDir ? ErrorCode.ENOTDIR : ErrorCode.EISDIR, p); + /* A path or URL supplied to the API was malformed. */ + case "EncodingError": + /* An operation depended on state cached in an interface object, but that state that has changed + since it was read from disk. */ + case "InvalidStateError": + /* The user attempted to write to a file or directory which could not be modified due to the state + of the underlying filesystem. */ + case "NoModificationAllowedError": + default: + return ApiError.FileError(ErrorCode.EINVAL, p); + } +} + // A note about getFile and getDirectory options: // These methods are called at numerous places in this file, and are passed // some combination of these two options: @@ -51,41 +95,34 @@ function _toArray(list?: any[]): any[] { // and throw an error if it does. export class HTML5FSFile extends PreloadFile implements IFile { - constructor(_fs: HTML5FS, _path: string, _flag: FileFlag, _stat: Stats, contents?: Buffer) { - super(_fs, _path, _flag, _stat, contents); + private _entry: FileEntry; + + constructor(fs: HTML5FS, entry: FileEntry, path: string, flag: FileFlag, stat: Stats, contents?: Buffer) { + super(fs, path, flag, stat, contents); + this._entry = entry; } public sync(cb: (e?: ApiError) => void): void { - if (this.isDirty()) { - // Don't create the file (it should already have been created by `open`) - let opts = { - create: false - }; - let _fs = this._fs; - let success: FileEntryCallback = (entry) => { - entry.createWriter((writer) => { - let buffer = this.getBuffer(); - let blob = new Blob([buffer2ArrayBuffer(buffer)]); - let length = blob.size; - writer.onwriteend = () => { - writer.onwriteend = null; - writer.truncate(length); - this.resetDirty(); - cb(); - }; - writer.onerror = (err: any) => { - cb(_fs.convert(err, this.getPath(), false)); - }; - writer.write(blob); - }); + if (!this.isDirty()) { + return cb(); + } + + this._entry.createWriter((writer) => { + let buffer = this.getBuffer(); + let blob = new Blob([buffer2ArrayBuffer(buffer)]); + let length = blob.size; + writer.onwriteend = (err?: any) => { + writer.onwriteend = null; + writer.onerror = null; + writer.truncate(length); + this.resetDirty(); + cb(); }; - let error = (err: DOMError) => { - cb(_fs.convert(err, this.getPath(), false)); + writer.onerror = (err: any) => { + cb(convertError(err, this.getPath(), false)); }; - _fs.fs.root.getFile(this.getPath(), opts, success, error); - } else { - cb(); - } + writer.write(blob); + }); } public close(cb: (e?: ApiError) => void): void { @@ -134,50 +171,6 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { return false; } - /** - * Converts the given DOMError into an appropriate ApiError. - * Full list of values here: - * https://developer.mozilla.org/en-US/docs/Web/API/DOMError - */ - public convert(err: DOMError, p: string, expectedDir: boolean): ApiError { - switch (err.name) { - /* The user agent failed to create a file or directory due to the existence of a file or - directory with the same path. */ - case "PathExistsError": - return ApiError.EEXIST(p); - /* The operation failed because it would cause the application to exceed its storage quota. */ - case 'QuotaExceededError': - return ApiError.FileError(ErrorCode.ENOSPC, p); - /* A required file or directory could not be found at the time an operation was processed. */ - case 'NotFoundError': - return ApiError.ENOENT(p); - /* This is a security error code to be used in situations not covered by any other error codes. - - A required file was unsafe for access within a Web application - - Too many calls are being made on filesystem resources */ - case 'SecurityError': - return ApiError.FileError(ErrorCode.EACCES, p); - /* The modification requested was illegal. Examples of invalid modifications include moving a - directory into its own child, moving a file into its parent directory without changing its name, - or copying a directory to a path occupied by a file. */ - case 'InvalidModificationError': - return ApiError.FileError(ErrorCode.EPERM, p); - /* The user has attempted to look up a file or directory, but the Entry found is of the wrong type - [e.g. is a DirectoryEntry when the user requested a FileEntry]. */ - case 'TypeMismatchError': - return ApiError.FileError(expectedDir ? ErrorCode.ENOTDIR : ErrorCode.EISDIR, p); - /* A path or URL supplied to the API was malformed. */ - case "EncodingError": - /* An operation depended on state cached in an interface object, but that state that has changed - since it was read from disk. */ - case "InvalidStateError": - /* The user attempted to write to a file or directory which could not be modified due to the state - of the underlying filesystem. */ - case "NoModificationAllowedError": - default: - return ApiError.FileError(ErrorCode.EINVAL, p); - } - } - /** * Nonstandard * Requests a storage quota from the browser to back this FS. @@ -188,7 +181,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { cb(); }; let error = (err: DOMException): void => { - cb(this.convert(err, "/", true)); + cb(convertError(err, "/", true)); }; if (this.type === global.PERSISTENT) { _requestQuota(this.type, this.size, (granted: number) => { @@ -227,7 +220,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { cb(); }; let error = (err: DOMException) => { - cb(this.convert(err, entry.fullPath, !entry.isDirectory)); + cb(convertError(err, entry.fullPath, !entry.isDirectory)); }; if (isDirectoryEntry(entry)) { entry.removeRecursively(succ, error); @@ -249,7 +242,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { currentPath: string = oldPath, error = (err: DOMException): void => { if (--semaphore <= 0) { - cb(this.convert(err, currentPath, false)); + cb(convertError(err, currentPath, false)); } }, success = (file: Entry): void => { @@ -319,7 +312,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { }; // Called when the path couldn't be opened as a directory or a file. let failedToLoad = (err: DOMException): void => { - cb(this.convert(err, path, false /* Unknown / irrelevant */)); + cb(convertError(err, path, false /* Unknown / irrelevant */)); }; // Called when the path couldn't be opened as a file, but might still be a // directory. @@ -337,7 +330,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { if (err.name === 'InvalidModificationError' && flags.isExclusive()) { cb(ApiError.EEXIST(p)); } else { - cb(this.convert(err, p, false)); + cb(convertError(err, p, false)); } }; @@ -349,7 +342,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { entry.file((file: File): void => { let reader = new FileReader(); reader.onloadend = (event: Event): void => { - let bfsFile = this._makeFile(p, flags, file, reader.result); + let bfsFile = this._makeFile(p, entry, flags, file, reader.result); cb(null, bfsFile); }; reader.onerror = (ev: Event) => { @@ -388,7 +381,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { cb(); }; let error = (err: DOMException): void => { - cb(this.convert(err, path, true)); + cb(convertError(err, path, true)); }; this.fs.root.getDirectory(path, opts, success, error); } @@ -413,10 +406,10 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { * Returns a BrowserFS object representing a File, created from the data * returned by calls to the Dropbox API. */ - private _makeFile(path: string, flag: FileFlag, stat: File, data: ArrayBuffer = new ArrayBuffer(0)): HTML5FSFile { + private _makeFile(path: string, entry: FileEntry, flag: FileFlag, stat: File, data: ArrayBuffer = new ArrayBuffer(0)): HTML5FSFile { let stats = new Stats(FileType.FILE, stat.size); let buffer = arrayBuffer2Buffer(data); - return new HTML5FSFile(this, path, flag, stats, buffer); + return new HTML5FSFile(this, entry, path, flag, stats, buffer); } /** @@ -424,7 +417,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { */ private _readdir(path: string, cb: (e: ApiError, entries?: Entry[]) => void): void { let error = (err: DOMException): void => { - cb(this.convert(err, path, true)); + cb(convertError(err, path, true)); }; // Grab the requested directory. this.fs.root.getDirectory(path, { create: false }, (dirEntry: DirectoryEntry) => { @@ -458,12 +451,12 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem { cb(); }; let err = (err: DOMException) => { - cb(this.convert(err, path, !isFile)); + cb(convertError(err, path, !isFile)); }; entry.remove(succ, err); }; let error = (err: DOMException): void => { - cb(this.convert(err, path, !isFile)); + cb(convertError(err, path, !isFile)); }; // Deleting the entry, so don't create it let opts = { diff --git a/src/backend/ZipFS.ts b/src/backend/ZipFS.ts index 12d1bef0..91879ed1 100644 --- a/src/backend/ZipFS.ts +++ b/src/backend/ZipFS.ts @@ -55,6 +55,11 @@ const inflateRaw: { } = require('pako/lib/inflate').inflateRaw; import {FileIndex, DirInode, FileInode, isDirInode, isFileInode} from '../generic/file_index'; +/** + * Maps CompressionMethod => function that decompresses. + */ +const decompressionMethods: {[method: number]: (data: Buffer, compressedSize: number, uncompressedSize: number) => Buffer} = {}; + /** * 4.4.2.2: Indicates the compatibiltiy of a file's external attributes. */ @@ -228,21 +233,16 @@ export class FileData { constructor(private header: FileHeader, private record: CentralDirectory, private data: Buffer) {} public decompress(): Buffer { // Check the compression - let compressionMethod: CompressionMethod = this.header.compressionMethod(); - switch (compressionMethod) { - case CompressionMethod.DEFLATE: - let data = inflateRaw( - this.data.slice(0, this.record.compressedSize()), - { chunkSize: this.record.uncompressedSize() } - ); - return arrayish2Buffer(data); - case CompressionMethod.STORED: - // Grab and copy. - return copyingSlice(this.data, 0, this.record.uncompressedSize()); - default: - let name: string = CompressionMethod[compressionMethod]; - name = name ? name : "Unknown: " + compressionMethod; - throw new ApiError(ErrorCode.EINVAL, "Invalid compression method on file '" + this.header.fileName() + "': " + name); + const compressionMethod: CompressionMethod = this.header.compressionMethod(); + const fcn = decompressionMethods[compressionMethod]; + if (fcn) { + return fcn(this.data, this.record.compressedSize(), this.record.uncompressedSize()); + } else { + let name: string = CompressionMethod[compressionMethod]; + if (!name) { + name = `Unknown: ${compressionMethod}`; + } + throw new ApiError(ErrorCode.EINVAL, `Invalid compression method on file '${this.header.fileName()}': ${name}`); } } public getHeader(): FileHeader { @@ -503,8 +503,16 @@ export class ZipTOC { } export default class ZipFS extends SynchronousFileSystem implements FileSystem { + /* tslint:disable:variable-name */ + public static readonly CompressionMethod = CompressionMethod; + /* tslint:enable:variable-name */ + public static isAvailable(): boolean { return true; } + public static RegisterDecompressionMethod(m: CompressionMethod, fcn: (data: Buffer, compressedSize: number, uncompressedSize: number) => Buffer): void { + decompressionMethods[m] = fcn; + } + public static computeIndex(data: Buffer, cb: (zipTOC: ZipTOC) => void) { const index: FileIndex = new FileIndex(); const eocd: EndOfCentralDirectory = ZipFS.getEOCD(data); @@ -753,3 +761,14 @@ export default class ZipFS extends SynchronousFileSystem implements FileSystem { } } } + +ZipFS.RegisterDecompressionMethod(CompressionMethod.DEFLATE, (data, compressedSize, uncompressedSize) => { + return arrayish2Buffer(inflateRaw( + data.slice(0, compressedSize), + { chunkSize: uncompressedSize } + )); +}); + +ZipFS.RegisterDecompressionMethod(CompressionMethod.STORED, (data, compressedSize, uncompressedSize) => { + return copyingSlice(data, 0, uncompressedSize); +});