Skip to content

Commit

Permalink
Faster builds, HTML5FS optimization, support ZipFS decompression plug…
Browse files Browse the repository at this point in the history 
…ins.
  • Loading branch information
John Vilk committed Nov 2, 2016
1 parent a37b5e3 commit 3f7a60a
Show file tree
Hide file tree
Showing 3 changed files with 147 additions and 106 deletions.
39 changes: 34 additions & 5 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
"karma-opera-launcher": "^1.0.0",
"karma-safari-launcher": "^1.0.0",
"mocha": "^3.1.1",
"npm-run-all": "^3.1.1",
"object-wrapper": "^0.2.0",
"remap-istanbul": "^0.6.4",
"rimraf": "^2.5.4",
Expand All @@ -68,11 +69,39 @@
},
"scripts": {
"lint": "tslint -c src/tslint.json --project src/tsconfig.json",
"build": "tsc -p src && rollup -c src/rollup.config.js && webpack --config src/webpack.config.js && webpack -p --config src/webpack.config.js",
"build_tests": "tsc -p test && rollup -c test/rollup.config.js && rollup -c test/rollup.worker.config.js && webpack --config test/webpack.config.js",
"build_scripts": "tsc -p scripts",
"dist": "npm run build && npm run build_scripts && npm run lint && node build/scripts/make_dist.js && tsc -p src/tsconfig.node.json",
"test": "npm run build_scripts && node build/scripts/make_fixture_loader.js && node build/scripts/make_test_launcher.js && npm run build_tests && node build/scripts/make_zip_fixtures.js && node build/scripts/make_xhrfs_index.js test/fixtures/xhrfs/listings.json && karma start karma.config.js",
"build:tsc": "tsc -p src",
"watch:tsc": "tsc -p src --watch",
"build:scripts": "tsc -p scripts",
"watch:scripts": "tsc -p scripts --watch",
"build:rollup": "rollup -c src/rollup.config.js",
"watch:rollup": "rollup -w -c src/rollup.config.js",
"build:webpack": "webpack --config src/webpack.config.js",
"watch:webpack": "webpack -w --config src/webpack.config.js",
"build:webpack-release": "webpack -p --config src/webpack.config.js",
"watch:webpack-release": "webpack -w -p --config src/webpack.config.js",
"build": "npm-run-all --parallel build:tsc build:scripts --sequential build:rollup --parallel build:webpack build:webpack-release",
"watch": "npm-run-all build --parallel watch:tsc watch:scripts watch:rollup watch:webpack watch:webpack-release",
"test:build:tsc": "tsc -p test",
"test:watch:tsc": "tsc --watch -p test",
"test:build:rollup": "rollup -c test/rollup.config.js",
"test:watch:rollup": "rollup -w -c test/rollup.config.js",
"test:build:rollup-worker": "rollup -c test/rollup.worker.config.js",
"test:watch:rollup-worker": "rollup -w -c test/rollup.worker.config.js",
"test:build:webpack": "webpack --config test/webpack.config.js",
"test:watch:webpack": "webpack -w --config test/webpack.config.js",
"test:build": "npm-run-all test:build:tsc --parallel test:build:rollup test:build:rollup-worker --sequential test:build:webpack",
"test:watch": "npm-run-all --parallel test:watch:tsc test:watch:rollup test:watch:rollup-worker test:watch:webpack",
"dist:build:node": "tsc -p src/tsconfig.node.json",
"script:make_dist": "node build/scripts/make_dist.js",
"dist": "npm-run-all build lint script:make_dist dist:build:node",
"script:make_fixture_loader": "node build/scripts/make_fixture_loader.js",
"script:make_test_launcher": "node build/scripts/make_test_launcher.js",
"script:make_zip_fixtures": "node build/scripts/make_zip_fixtures",
"script:make_xhrfs_index": "node build/scripts/make_xhrfs_index.js test/fixtures/xhrfs/listings.json",
"test:karma": "karma start karma.config.js",
"test:prepare": "npm-run-all build:scripts script:make_fixture_loader script:make_test_launcher test:build script:make_zip_fixtures script:make_xhrfs_index",
"test": "npm-run-all test:prepare test:karma",
"watch-test": "npm-run-all test:prepare --parallel watch:scripts test:watch test:karma",
"prepublish": "npm run dist"
},
"dependencies": {
Expand Down
165 changes: 79 additions & 86 deletions src/backend/HTML5FS.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,50 @@ function _toArray(list?: any[]): any[] {
return Array.prototype.slice.call(list || [], 0);
}

/**
* Converts the given DOMError into an appropriate ApiError.
* Full list of values here:
* https://developer.mozilla.org/en-US/docs/Web/API/DOMError
*/
function convertError(err: DOMError, p: string, expectedDir: boolean): ApiError {
switch (err.name) {
/* The user agent failed to create a file or directory due to the existence of a file or
directory with the same path. */
case "PathExistsError":
return ApiError.EEXIST(p);
/* The operation failed because it would cause the application to exceed its storage quota. */
case 'QuotaExceededError':
return ApiError.FileError(ErrorCode.ENOSPC, p);
/* A required file or directory could not be found at the time an operation was processed. */
case 'NotFoundError':
return ApiError.ENOENT(p);
/* This is a security error code to be used in situations not covered by any other error codes.
- A required file was unsafe for access within a Web application
- Too many calls are being made on filesystem resources */
case 'SecurityError':
return ApiError.FileError(ErrorCode.EACCES, p);
/* The modification requested was illegal. Examples of invalid modifications include moving a
directory into its own child, moving a file into its parent directory without changing its name,
or copying a directory to a path occupied by a file. */
case 'InvalidModificationError':
return ApiError.FileError(ErrorCode.EPERM, p);
/* The user has attempted to look up a file or directory, but the Entry found is of the wrong type
[e.g. is a DirectoryEntry when the user requested a FileEntry]. */
case 'TypeMismatchError':
return ApiError.FileError(expectedDir ? ErrorCode.ENOTDIR : ErrorCode.EISDIR, p);
/* A path or URL supplied to the API was malformed. */
case "EncodingError":
/* An operation depended on state cached in an interface object, but that state that has changed
since it was read from disk. */
case "InvalidStateError":
/* The user attempted to write to a file or directory which could not be modified due to the state
of the underlying filesystem. */
case "NoModificationAllowedError":
default:
return ApiError.FileError(ErrorCode.EINVAL, p);
}
}

// A note about getFile and getDirectory options:
// These methods are called at numerous places in this file, and are passed
// some combination of these two options:
Expand All @@ -51,41 +95,34 @@ function _toArray(list?: any[]): any[] {
// and throw an error if it does.

export class HTML5FSFile extends PreloadFile<HTML5FS> implements IFile {
constructor(_fs: HTML5FS, _path: string, _flag: FileFlag, _stat: Stats, contents?: Buffer) {
super(_fs, _path, _flag, _stat, contents);
private _entry: FileEntry;

constructor(fs: HTML5FS, entry: FileEntry, path: string, flag: FileFlag, stat: Stats, contents?: Buffer) {
super(fs, path, flag, stat, contents);
this._entry = entry;
}

public sync(cb: (e?: ApiError) => void): void {
if (this.isDirty()) {
// Don't create the file (it should already have been created by `open`)
let opts = {
create: false
};
let _fs = this._fs;
let success: FileEntryCallback = (entry) => {
entry.createWriter((writer) => {
let buffer = this.getBuffer();
let blob = new Blob([buffer2ArrayBuffer(buffer)]);
let length = blob.size;
writer.onwriteend = () => {
writer.onwriteend = null;
writer.truncate(length);
this.resetDirty();
cb();
};
writer.onerror = (err: any) => {
cb(_fs.convert(err, this.getPath(), false));
};
writer.write(blob);
});
if (!this.isDirty()) {
return cb();
}

this._entry.createWriter((writer) => {
let buffer = this.getBuffer();
let blob = new Blob([buffer2ArrayBuffer(buffer)]);
let length = blob.size;
writer.onwriteend = (err?: any) => {
writer.onwriteend = null;
writer.onerror = null;
writer.truncate(length);
this.resetDirty();
cb();
};
let error = (err: DOMError) => {
cb(_fs.convert(err, this.getPath(), false));
writer.onerror = (err: any) => {
cb(convertError(err, this.getPath(), false));
};
_fs.fs.root.getFile(this.getPath(), opts, success, error);
} else {
cb();
}
writer.write(blob);
});
}

public close(cb: (e?: ApiError) => void): void {
Expand Down Expand Up @@ -134,50 +171,6 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem {
return false;
}

/**
* Converts the given DOMError into an appropriate ApiError.
* Full list of values here:
* https://developer.mozilla.org/en-US/docs/Web/API/DOMError
*/
public convert(err: DOMError, p: string, expectedDir: boolean): ApiError {
switch (err.name) {
/* The user agent failed to create a file or directory due to the existence of a file or
directory with the same path. */
case "PathExistsError":
return ApiError.EEXIST(p);
/* The operation failed because it would cause the application to exceed its storage quota. */
case 'QuotaExceededError':
return ApiError.FileError(ErrorCode.ENOSPC, p);
/* A required file or directory could not be found at the time an operation was processed. */
case 'NotFoundError':
return ApiError.ENOENT(p);
/* This is a security error code to be used in situations not covered by any other error codes.
- A required file was unsafe for access within a Web application
- Too many calls are being made on filesystem resources */
case 'SecurityError':
return ApiError.FileError(ErrorCode.EACCES, p);
/* The modification requested was illegal. Examples of invalid modifications include moving a
directory into its own child, moving a file into its parent directory without changing its name,
or copying a directory to a path occupied by a file. */
case 'InvalidModificationError':
return ApiError.FileError(ErrorCode.EPERM, p);
/* The user has attempted to look up a file or directory, but the Entry found is of the wrong type
[e.g. is a DirectoryEntry when the user requested a FileEntry]. */
case 'TypeMismatchError':
return ApiError.FileError(expectedDir ? ErrorCode.ENOTDIR : ErrorCode.EISDIR, p);
/* A path or URL supplied to the API was malformed. */
case "EncodingError":
/* An operation depended on state cached in an interface object, but that state that has changed
since it was read from disk. */
case "InvalidStateError":
/* The user attempted to write to a file or directory which could not be modified due to the state
of the underlying filesystem. */
case "NoModificationAllowedError":
default:
return ApiError.FileError(ErrorCode.EINVAL, p);
}
}

/**
* Nonstandard
* Requests a storage quota from the browser to back this FS.
Expand All @@ -188,7 +181,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem {
cb();
};
let error = (err: DOMException): void => {
cb(this.convert(err, "/", true));
cb(convertError(err, "/", true));
};
if (this.type === global.PERSISTENT) {
_requestQuota(this.type, this.size, (granted: number) => {
Expand Down Expand Up @@ -227,7 +220,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem {
cb();
};
let error = (err: DOMException) => {
cb(this.convert(err, entry.fullPath, !entry.isDirectory));
cb(convertError(err, entry.fullPath, !entry.isDirectory));
};
if (isDirectoryEntry(entry)) {
entry.removeRecursively(succ, error);
Expand All @@ -249,7 +242,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem {
currentPath: string = oldPath,
error = (err: DOMException): void => {
if (--semaphore <= 0) {
cb(this.convert(err, currentPath, false));
cb(convertError(err, currentPath, false));
}
},
success = (file: Entry): void => {
Expand Down Expand Up @@ -319,7 +312,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem {
};
// Called when the path couldn't be opened as a directory or a file.
let failedToLoad = (err: DOMException): void => {
cb(this.convert(err, path, false /* Unknown / irrelevant */));
cb(convertError(err, path, false /* Unknown / irrelevant */));
};
// Called when the path couldn't be opened as a file, but might still be a
// directory.
Expand All @@ -337,7 +330,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem {
if (err.name === 'InvalidModificationError' && flags.isExclusive()) {
cb(ApiError.EEXIST(p));
} else {
cb(this.convert(err, p, false));
cb(convertError(err, p, false));
}
};

Expand All @@ -349,7 +342,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem {
entry.file((file: File): void => {
let reader = new FileReader();
reader.onloadend = (event: Event): void => {
let bfsFile = this._makeFile(p, flags, file, <ArrayBuffer> reader.result);
let bfsFile = this._makeFile(p, entry, flags, file, <ArrayBuffer> reader.result);
cb(null, bfsFile);
};
reader.onerror = (ev: Event) => {
Expand Down Expand Up @@ -388,7 +381,7 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem {
cb();
};
let error = (err: DOMException): void => {
cb(this.convert(err, path, true));
cb(convertError(err, path, true));
};
this.fs.root.getDirectory(path, opts, success, error);
}
Expand All @@ -413,18 +406,18 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem {
* Returns a BrowserFS object representing a File, created from the data
* returned by calls to the Dropbox API.
*/
private _makeFile(path: string, flag: FileFlag, stat: File, data: ArrayBuffer = new ArrayBuffer(0)): HTML5FSFile {
private _makeFile(path: string, entry: FileEntry, flag: FileFlag, stat: File, data: ArrayBuffer = new ArrayBuffer(0)): HTML5FSFile {
let stats = new Stats(FileType.FILE, stat.size);
let buffer = arrayBuffer2Buffer(data);
return new HTML5FSFile(this, path, flag, stats, buffer);
return new HTML5FSFile(this, entry, path, flag, stats, buffer);
}

/**
* Returns an array of `FileEntry`s. Used internally by empty and readdir.
*/
private _readdir(path: string, cb: (e: ApiError, entries?: Entry[]) => void): void {
let error = (err: DOMException): void => {
cb(this.convert(err, path, true));
cb(convertError(err, path, true));
};
// Grab the requested directory.
this.fs.root.getDirectory(path, { create: false }, (dirEntry: DirectoryEntry) => {
Expand Down Expand Up @@ -458,12 +451,12 @@ export default class HTML5FS extends BaseFileSystem implements IFileSystem {
cb();
};
let err = (err: DOMException) => {
cb(this.convert(err, path, !isFile));
cb(convertError(err, path, !isFile));
};
entry.remove(succ, err);
};
let error = (err: DOMException): void => {
cb(this.convert(err, path, !isFile));
cb(convertError(err, path, !isFile));
};
// Deleting the entry, so don't create it
let opts = {
Expand Down
49 changes: 34 additions & 15 deletions src/backend/ZipFS.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ const inflateRaw: {
} = require('pako/lib/inflate').inflateRaw;
import {FileIndex, DirInode, FileInode, isDirInode, isFileInode} from '../generic/file_index';

/**
* Maps CompressionMethod => function that decompresses.
*/
const decompressionMethods: {[method: number]: (data: Buffer, compressedSize: number, uncompressedSize: number) => Buffer} = {};

/**
* 4.4.2.2: Indicates the compatibiltiy of a file's external attributes.
*/
Expand Down Expand Up @@ -228,21 +233,16 @@ export class FileData {
constructor(private header: FileHeader, private record: CentralDirectory, private data: Buffer) {}
public decompress(): Buffer {
// Check the compression
let compressionMethod: CompressionMethod = this.header.compressionMethod();
switch (compressionMethod) {
case CompressionMethod.DEFLATE:
let data = inflateRaw(
this.data.slice(0, this.record.compressedSize()),
{ chunkSize: this.record.uncompressedSize() }
);
return arrayish2Buffer(data);
case CompressionMethod.STORED:
// Grab and copy.
return copyingSlice(this.data, 0, this.record.uncompressedSize());
default:
let name: string = CompressionMethod[compressionMethod];
name = name ? name : "Unknown: " + compressionMethod;
throw new ApiError(ErrorCode.EINVAL, "Invalid compression method on file '" + this.header.fileName() + "': " + name);
const compressionMethod: CompressionMethod = this.header.compressionMethod();
const fcn = decompressionMethods[compressionMethod];
if (fcn) {
return fcn(this.data, this.record.compressedSize(), this.record.uncompressedSize());
} else {
let name: string = CompressionMethod[compressionMethod];
if (!name) {
name = `Unknown: ${compressionMethod}`;
}
throw new ApiError(ErrorCode.EINVAL, `Invalid compression method on file '${this.header.fileName()}': ${name}`);
}
}
public getHeader(): FileHeader {
Expand Down Expand Up @@ -503,8 +503,16 @@ export class ZipTOC {
}

export default class ZipFS extends SynchronousFileSystem implements FileSystem {
/* tslint:disable:variable-name */
public static readonly CompressionMethod = CompressionMethod;
/* tslint:enable:variable-name */

public static isAvailable(): boolean { return true; }

public static RegisterDecompressionMethod(m: CompressionMethod, fcn: (data: Buffer, compressedSize: number, uncompressedSize: number) => Buffer): void {
decompressionMethods[m] = fcn;
}

public static computeIndex(data: Buffer, cb: (zipTOC: ZipTOC) => void) {
const index: FileIndex<CentralDirectory> = new FileIndex<CentralDirectory>();
const eocd: EndOfCentralDirectory = ZipFS.getEOCD(data);
Expand Down Expand Up @@ -753,3 +761,14 @@ export default class ZipFS extends SynchronousFileSystem implements FileSystem {
}
}
}

ZipFS.RegisterDecompressionMethod(CompressionMethod.DEFLATE, (data, compressedSize, uncompressedSize) => {
return arrayish2Buffer(inflateRaw(
data.slice(0, compressedSize),
{ chunkSize: uncompressedSize }
));
});

ZipFS.RegisterDecompressionMethod(CompressionMethod.STORED, (data, compressedSize, uncompressedSize) => {
return copyingSlice(data, 0, uncompressedSize);
});

0 comments on commit 3f7a60a

Please sign in to comment.