Skip to content

Commit

Permalink
[monaco] stream content to avoid blocking the backend and network by …
Browse files Browse the repository at this point in the history 
…large files

Signed-off-by: Anton Kosyakov <anton.kosyakov@typefox.io>
  • Loading branch information
@akosyakov
akosyakov committed Jul 10, 2020
1 parent b1b385c commit 4041d16
Show file tree
Hide file tree
Showing 6 changed files with 167 additions and 28 deletions.
39 changes: 33 additions & 6 deletions packages/core/src/common/resource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import { Disposable } from './disposable';
import { MaybePromise } from './types';
import { CancellationToken } from './cancellation';
import { ApplicationError } from './application-error';
import { ReadableStream, Readable } from './stream';

export interface ResourceVersion {
}
Expand Down Expand Up @@ -62,6 +63,15 @@ export interface Resource extends Disposable {
* @throws `ResourceError.NotFound` if a resource not found
*/
readContents(options?: ResourceReadOptions): Promise<string>;
/**
* Stream latest content of this resource.
*
* If a resource supports versioning it updates version to latest.
* If a resource supports encoding it updates encoding to latest.
*
* @throws `ResourceError.NotFound` if a resource not found
*/
readStream?(options?: ResourceReadOptions): Promise<ReadableStream<string>>;
/**
* Rewrites the complete content for this resource.
* If a resource does not exist it will be created.
Expand All @@ -74,6 +84,18 @@ export interface Resource extends Disposable {
* @throws `ResourceError.OutOfSync` if latest resource version is out of sync with the given
*/
saveContents?(content: string, options?: ResourceSaveOptions): Promise<void>;
/**
* Rewrites the complete content for this resource.
* If a resource does not exist it will be created.
*
* If a resource supports versioning clients can pass some version
* to check against it, if it is not provided latest version is used.
*
* It updates version and encoding to latest.
*
* @throws `ResourceError.OutOfSync` if latest resource version is out of sync with the given
*/
saveStream?(content: Readable<string>, options?: ResourceSaveOptions): Promise<void>;
/**
* Applies incremental content changes to this resource.
*
Expand All @@ -90,7 +112,8 @@ export interface Resource extends Disposable {
}
export namespace Resource {
export interface SaveContext {
content: string
contentLength: number
content: string | Readable<string>
changes?: TextDocumentContentChangeEvent[]
options?: ResourceSaveOptions
}
Expand All @@ -104,10 +127,15 @@ export namespace Resource {
if (token && token.isCancellationRequested) {
return;
}
await resource.saveContents(context.content, context.options);
if (typeof context.content !== 'string' && resource.saveStream) {
await resource.saveStream(context.content, context.options);
} else {
const content = typeof context.content === 'string' ? context.content : Readable.toString(context.content);
await resource.saveContents(content, context.options);
}
}
export async function trySaveContentChanges(resource: Resource, context: SaveContext): Promise<boolean> {
if (!context.changes || !resource.saveContentChanges || shouldSaveContent(context)) {
if (!context.changes || !resource.saveContentChanges || shouldSaveContent(resource, context)) {
return false;
}
try {
Expand All @@ -120,12 +148,11 @@ export namespace Resource {
return false;
}
}
export function shouldSaveContent({ content, changes }: SaveContext): boolean {
if (!changes) {
export function shouldSaveContent(resource: Resource, { contentLength, changes }: SaveContext): boolean {
if (!changes || (resource.saveStream && contentLength > 32 * 1024 * 1024)) {
return true;
}
let contentChangesLength = 0;
const contentLength = content.length;
for (const change of changes) {
contentChangesLength += JSON.stringify(change).length;
if (contentChangesLength > contentLength) {
Expand Down
8 changes: 8 additions & 0 deletions packages/core/src/common/stream.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ export namespace Readable {
}
};
}
export function toString(readable: Readable<string>): string {
let result = '';
let chunk: string | null;
while ((chunk = readable.read()) != null) {
result += chunk;
}
return result;
}
}

/**
Expand Down
57 changes: 55 additions & 2 deletions packages/filesystem/src/browser/file-resource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { injectable, inject } from 'inversify';
import { Resource, ResourceVersion, ResourceResolver, ResourceError, ResourceSaveOptions } from '@theia/core/lib/common/resource';
import { DisposableCollection } from '@theia/core/lib/common/disposable';
import { Emitter, Event } from '@theia/core/lib/common/event';
import { Readable, ReadableStream } from '@theia/core/lib/common/stream';
import URI from '@theia/core/lib/common/uri';
import { FileOperation, FileOperationError, FileOperationResult, ETAG_DISABLED, FileSystemProviderCapabilities, FileReadStreamOptions, BinarySize } from '../common/files';
import { FileService, TextFileOperationError, TextFileOperationResult } from './file-service';
Expand Down Expand Up @@ -135,7 +136,59 @@ export class FileResource implements Resource {
}
}

async saveContents(content: string, options?: ResourceSaveOptions): Promise<void> {
async readStream(options?: { encoding?: string }): Promise<ReadableStream<string>> {
try {
const encoding = options?.encoding || this.version?.encoding;
const stat = await this.fileService.readStream(this.uri, {
encoding,
etag: ETAG_DISABLED,
acceptTextOnly: this.acceptTextOnly,
limits: this.limits
});
this._version = {
encoding: stat.encoding,
etag: stat.etag,
mtime: stat.mtime
};
return stat.value;
} catch (e) {
if (e instanceof TextFileOperationError && e.textFileOperationResult === TextFileOperationResult.FILE_IS_BINARY) {
if (await this.shouldOpenAsText('The file is either binary or uses an unsupported text encoding.')) {
this.acceptTextOnly = false;
return this.readStream(options);
}
} else if (e instanceof FileOperationError && e.fileOperationResult === FileOperationResult.FILE_TOO_LARGE) {
const stat = await this.fileService.resolve(this.uri, { resolveMetadata: true });
const maxFileSize = GENERAL_MAX_FILE_SIZE_MB * 1024 * 1024;
if (this.limits?.size !== maxFileSize && await this.shouldOpenAsText(`The file is too large (${BinarySize.formatSize(stat.size)}).`)) {
this.limits = {
size: maxFileSize
};
return this.readStream(options);
}
} else if (e instanceof FileOperationError && e.fileOperationResult === FileOperationResult.FILE_NOT_FOUND) {
this._version = undefined;
const { message, stack } = e;
throw ResourceError.NotFound({
message, stack,
data: {
uri: this.uri
}
});
}
throw e;
}
}

saveContents(content: string, options?: ResourceSaveOptions): Promise<void> {
return this.doWrite(content, options);
}

saveStream(content: Readable<string>, options?: ResourceSaveOptions): Promise<void> {
return this.doWrite(content, options);
}

protected async doWrite(content: string | Readable<string>, options?: ResourceSaveOptions): Promise<void> {
const version = options?.version || this._version;
const current = FileResourceVersion.is(version) ? version : undefined;
const etag = current?.etag;
Expand All @@ -154,7 +207,7 @@ export class FileResource implements Resource {
} catch (e) {
if (e instanceof FileOperationError && e.fileOperationResult === FileOperationResult.FILE_MODIFIED_SINCE) {
if (etag !== ETAG_DISABLED && await this.shouldOverwrite()) {
return this.saveContents(content, { ...options, version: { stat: { ...current, etag: ETAG_DISABLED } } });
return this.doWrite(content, { ...options, version: { stat: { ...current, etag: ETAG_DISABLED } } });
}
const { message, stack } = e;
throw ResourceError.OutOfSync({ message, stack, data: { uri: this.uri } });
Expand Down
54 changes: 36 additions & 18 deletions packages/monaco/src/browser/monaco-editor-model.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,21 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument {
* Only this method can create an instance of `monaco.editor.IModel`,
* there should not be other calls to `monaco.editor.createModel`.
*/
protected initialize(content: string): void {
protected initialize(value: string | monaco.editor.ITextBufferFactory): void {
if (!this.toDispose.disposed) {
this.model = monaco.editor.createModel(content, undefined, monaco.Uri.parse(this.resource.uri.toString()));
const uri = monaco.Uri.parse(this.resource.uri.toString());
let firstLine;
if (typeof value === 'string') {
firstLine = value;
const firstLF = value.indexOf('\n');
if (firstLF !== -1) {
firstLine = value.substring(0, firstLF);
}
} else {
firstLine = value.getFirstLineText(1000);
}
const languageSelection = monaco.services.StaticServices.modeService.get().createByFilepathOrFirstLine(uri, firstLine);
this.model = monaco.services.StaticServices.modelService.get().createModel(value, languageSelection, uri);
this.resourceVersion = this.resource.version;
this.updateSavedVersionId();
this.toDispose.push(this.model);
Expand Down Expand Up @@ -311,29 +323,30 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument {
return;
}

const newText = await this.readContents();
if (newText === undefined || token.isCancellationRequested || this._dirty) {
return;
}
this.resourceVersion = this.resource.version;

const value = this.model.getValue();
if (value === newText) {
const value = await this.readContents();
if (value === undefined || token.isCancellationRequested || this._dirty) {
return;
}

const range = this.m2p.asRange(this.model.getFullModelRange());
this.applyEdits([this.p2m.asTextEdit({ range, newText }) as monaco.editor.IIdentifiedSingleEditOperation], {
this.resourceVersion = this.resource.version;
this.updateModel(() => monaco.services.StaticServices.modelService.get().updateModel(this.model, value), {
ignoreDirty: true,
ignoreContentChanges: true
});
}
protected async readContents(): Promise<string | undefined> {
protected async readContents(): Promise<string | monaco.editor.ITextBufferFactory | undefined> {
try {
const content = await this.resource.readContents({ encoding: this.getEncoding() });
const options = { encoding: this.getEncoding() };
const content = await (this.resource.readStream ? this.resource.readStream(options) : this.resource.readContents(options));
let value;
if (typeof content === 'string') {
value = content;
} else {
value = monaco.textModel.createTextBufferFactoryFromStream(content);
}
this.updateContentEncoding();
this.setValid(true);
return content;
return value;
} catch (e) {
this.setValid(false);
if (ResourceError.NotFound.is(e)) {
Expand Down Expand Up @@ -411,6 +424,10 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument {
operations: monaco.editor.IIdentifiedSingleEditOperation[],
options?: Partial<MonacoEditorModel.ApplyEditsOptions>
): monaco.editor.IIdentifiedSingleEditOperation[] {
return this.updateModel(() => this.model.applyEdits(operations), options);
}

protected updateModel<T>(doUpdate: () => T, options?: Partial<MonacoEditorModel.ApplyEditsOptions>): T {
const resolvedOptions: MonacoEditorModel.ApplyEditsOptions = {
ignoreDirty: false,
ignoreContentChanges: false,
Expand All @@ -420,7 +437,7 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument {
this.ignoreDirtyEdits = resolvedOptions.ignoreDirty;
this.ignoreContentChanges = resolvedOptions.ignoreContentChanges;
try {
return this.model.applyEdits(operations);
return doUpdate();
} finally {
this.ignoreDirtyEdits = ignoreDirtyEdits;
this.ignoreContentChanges = ignoreContentChanges;
Expand All @@ -442,11 +459,12 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument {
return;
}

const content = this.model.getValue();
const contentLength = this.model.getValueLength();
const content = this.model.createSnapshot() || this.model.getValue();
try {
const encoding = this.getEncoding();
const version = this.resourceVersion;
await Resource.save(this.resource, { changes, content, options: { encoding, overwriteEncoding, version } }, token);
await Resource.save(this.resource, { changes, content, contentLength, options: { encoding, overwriteEncoding, version } }, token);
this.contentChanges.splice(0, changes.length);
this.resourceVersion = this.resource.version;
this.updateContentEncoding();
Expand Down
6 changes: 4 additions & 2 deletions packages/monaco/src/browser/monaco-loader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ export function loadMonaco(vsRequire: any): Promise<void> {
'vs/platform/contextkey/common/contextkey',
'vs/platform/contextkey/browser/contextKeyService',
'vs/editor/common/model/wordHelper',
'vs/base/common/errors'
'vs/base/common/errors',
'vs/editor/common/model/textModel',
], (commands: any, actions: any,
keybindingsRegistry: any, keybindingResolver: any, resolvedKeybinding: any, keybindingLabels: any,
keyCodes: any, mime: any, editorExtensions: any, simpleServices: any,
Expand All @@ -89,7 +90,7 @@ export function loadMonaco(vsRequire: any): Promise<void> {
markerService: any,
contextKey: any, contextKeyService: any,
wordHelper: any,
error: any) => {
error: any, textModel: any) => {
const global: any = self;
global.monaco.commands = commands;
global.monaco.actions = actions;
Expand All @@ -111,6 +112,7 @@ export function loadMonaco(vsRequire: any): Promise<void> {
global.monaco.mime = mime;
global.monaco.wordHelper = wordHelper;
global.monaco.error = error;
global.monaco.textModel = textModel;
resolve();
});
});
Expand Down
31 changes: 31 additions & 0 deletions packages/monaco/src/typings/monaco/index.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,24 @@ declare module monaco.instantiation {
}
}

declare module monaco.textModel {
interface ITextStream {
on(event: 'data', callback: (data: string) => void): void;
on(event: 'error', callback: (err: Error) => void): void;
on(event: 'end', callback: () => void): void;
on(event: string, callback: any): void;
}
// https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model/textModel.ts#L58
export function createTextBufferFactoryFromStream(stream: ITextStream, filter?: (chunk: any) => string, validator?: (chunk: any) => Error | undefined): Promise<monaco.editor.ITextBufferFactory>;
}

declare module monaco.editor {

// https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model.ts#L1263
export interface ITextBufferFactory {
getFirstLineText(lengthLimit: number): string;
}

export interface ICodeEditor {
protected readonly _instantiationService: monaco.instantiation.IInstantiationService;

Expand Down Expand Up @@ -344,6 +360,11 @@ declare module monaco.editor {
after?: IContentDecorationRenderOptions;
}

// https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model.ts#L522
export interface ITextSnapshot {
read(): string | null;
}

export interface ITextModel {
/**
* Get the tokens for the line `lineNumber`.
Expand All @@ -359,6 +380,9 @@ declare module monaco.editor {
*/
// https://github.com/theia-ide/vscode/blob/standalone/0.19.x/src/vs/editor/common/model.ts#L806-L810
forceTokenization(lineNumber: number): void;

// https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model.ts#L623
createSnapshot(): ITextSnapshot | null;
}

}
Expand Down Expand Up @@ -718,6 +742,12 @@ declare module monaco.services {
read(filter?: { owner?: string; resource?: monaco.Uri; severities?: number, take?: number; }): editor.IMarker[];
}

// https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/services/modelService.ts#L18
export interface IModelService {
createModel(value: string | monaco.editor.ITextBufferFactory, languageSelection: ILanguageSelection | null, resource?: monaco.URI, isForSimpleWidget?: boolean): monaco.editor.ITextModel;
updateModel(model: monaco.editor.ITextModel, value: string | monaco.editor.ITextBufferFactory): void;
}

// https://github.com/theia-ide/vscode/blob/standalone/0.19.x/src/vs/editor/standalone/browser/standaloneServices.ts#L56
export module StaticServices {
export function init(overrides: monaco.editor.IEditorOverrideServices): [ServiceCollection, monaco.instantiation.IInstantiationService];
Expand All @@ -728,6 +758,7 @@ declare module monaco.services {
export const resourcePropertiesService: LazyStaticService<ITextResourcePropertiesService>;
export const instantiationService: LazyStaticService<monaco.instantiation.IInstantiationService>;
export const markerService: LazyStaticService<IMarkerService>;
export const modelService: LazyStaticService<IModelService>;
}
}

Expand Down

0 comments on commit 4041d16

Please sign in to comment.