diff --git a/package.json b/package.json index 50ebfd5ab891a..76e6edf0a71db 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,8 @@ "hygiene": "node --max_old_space_size=4095 ./node_modules/gulp/bin/gulp.js hygiene", "core-ci": "node --max_old_space_size=4095 ./node_modules/gulp/bin/gulp.js core-ci", "extensions-ci": "node --max_old_space_size=4095 ./node_modules/gulp/bin/gulp.js extensions-ci", - "webview-generate-csp-hash": "npx github:apaatsio/csp-hash-from-html csp-hash ./src/vs/workbench/contrib/webview/browser/pre/index.html" + "webview-generate-csp-hash": "npx github:apaatsio/csp-hash-from-html csp-hash ./src/vs/workbench/contrib/webview/browser/pre/index.html", + "generate-tree-sitter-wasm": "npx tree-sitter build-wasm node_modules/tree-sitter-typescript/typescript" }, "dependencies": { "@microsoft/1ds-core-js": "^3.2.2", @@ -86,6 +87,7 @@ "vscode-proxy-agent": "^0.12.0", "vscode-regexpp": "^3.1.0", "vscode-textmate": "7.0.1", + "web-tree-sitter": "^0.20.7", "xterm": "5.1.0-beta.13", "xterm-addon-canvas": "0.3.0-beta.1", "xterm-addon-search": "0.11.0-beta.1", @@ -203,6 +205,8 @@ "source-map": "0.6.1", "source-map-support": "^0.3.2", "style-loader": "^1.3.0", + "tree-sitter-cli": "^0.20.6", + "tree-sitter-typescript": "^0.20.1", "ts-loader": "^9.2.7", "ts-node": "^10.9.1", "tsec": "0.1.4", diff --git a/src/vs/base/common/stopwatch.ts b/src/vs/base/common/stopwatch.ts index ccba2ac587410..10338b6b73193 100644 --- a/src/vs/base/common/stopwatch.ts +++ b/src/vs/base/common/stopwatch.ts @@ -10,8 +10,8 @@ const hasPerformanceNow = (globals.performance && typeof globals.performance.now export class StopWatch { private _highResolution: boolean; - private _startTime: number; - private _stopTime: number; + private _startTime!: number; + private _stopTime!: number; public static create(highResolution: boolean = true): StopWatch { return new StopWatch(highResolution); @@ -19,6 +19,10 @@ export class StopWatch { constructor(highResolution: boolean) { this._highResolution = hasPerformanceNow && highResolution; + this.reset(); + } + + public reset(): void { this._startTime = this._now(); this._stopTime = -1; } diff --git a/src/vs/editor/browser/services/treeSitterServices/tree-sitter-typescript.wasm b/src/vs/editor/browser/services/treeSitterServices/tree-sitter-typescript.wasm new file mode 100644 index 0000000000000..0ca4303523be5 Binary files /dev/null and b/src/vs/editor/browser/services/treeSitterServices/tree-sitter-typescript.wasm differ diff --git a/src/vs/editor/browser/services/treeSitterServices/treeSitterColorizationService.ts b/src/vs/editor/browser/services/treeSitterServices/treeSitterColorizationService.ts new file mode 100644 index 0000000000000..6c784f0e68a94 --- /dev/null +++ b/src/vs/editor/browser/services/treeSitterServices/treeSitterColorizationService.ts @@ -0,0 +1,123 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// eslint-disable-next-line local/code-import-patterns +import { init } from 'web-tree-sitter'; +import { registerSingleton } from 'vs/platform/instantiation/common/extensions'; +import { IModelService } from 'vs/editor/common/services/model'; +import { FileAccess } from 'vs/base/common/network'; +import { createDecorator, ServicesAccessor } from 'vs/platform/instantiation/common/instantiation'; +import { IThemeService } from 'vs/platform/theme/common/themeService'; +import { DisposableStore } from 'vs/base/common/lifecycle'; +import { TreeSitterColorizationTree } from 'vs/editor/browser/services/treeSitterTrees/colorization/treeSitterColorizationTree'; +import { Iterable } from 'vs/base/common/iterator'; +import { Action2, registerAction2 } from 'vs/platform/actions/common/actions'; +import { ITreeSitterService } from 'vs/editor/browser/services/treeSitterServices/treeSitterService'; +import { IFileService } from 'vs/platform/files/common/files'; + +const ITreeSitterColorizationService = createDecorator('ITreeSitterColorizationService'); + +export interface ITreeSitterColorizationService { + readonly _serviceBrand: undefined; + registerTreeSittersForColorization(asynchronous: boolean): void; + dispose(): void; + clearCache(): void; +} + +export class TreeSitterColorizationService implements ITreeSitterColorizationService { + + readonly _serviceBrand: undefined; + private readonly _disposableStore: DisposableStore = new DisposableStore(); + private readonly _treeSittersColorizationTrees: TreeSitterColorizationTree[] = []; + + constructor( + @ITreeSitterService private readonly _treeSitterService: ITreeSitterService, + @IModelService private readonly _modelService: IModelService, + @IThemeService private readonly _themeService: IThemeService, + @IFileService private readonly _fileService: IFileService + ) { } + + registerTreeSittersForColorization(asynchronous: boolean = true) { + this._initializeTreeSitterService(asynchronous); + const models = this._modelService.getModels(); + for (const model of models) { + if (model.getLanguageId() === 'typescript') { + model.tokenization.setTokens([]); + this._treeSittersColorizationTrees.push(new TreeSitterColorizationTree(model, this._treeSitterService, this._themeService, this._fileService, asynchronous)); + } + } + } + + private _initializeTreeSitterService(asynchronous: boolean = true) { + init({ + locateFile(_file: string, _folder: string) { + return FileAccess.asBrowserUri('../../../../../../node_modules/web-tree-sitter/tree-sitter.wasm', require).toString(true); + } + }).then(async () => { + this._disposableStore.add(this._modelService.onModelAdded((model) => { + if (model.getLanguageId() === 'typescript') { + model.tokenization.setTokens([]); + this._treeSittersColorizationTrees.push(new TreeSitterColorizationTree(model, this._treeSitterService, this._themeService, this._fileService, asynchronous)); + } + })); + this._disposableStore.add(this._modelService.onModelLanguageChanged((event) => { + const model = event.model; + if (model.getLanguageId() === 'typescript') { + model.tokenization.setTokens([]); + this._treeSittersColorizationTrees.push(new TreeSitterColorizationTree(model, this._treeSitterService, this._themeService, this._fileService, asynchronous)); + } + })); + this._disposableStore.add(this._modelService.onModelRemoved((model) => { + if (model.getLanguageId() === 'typescript') { + const treeSitterTreeToDispose = Iterable.find(this._treeSittersColorizationTrees, tree => tree.id === model.id); + if (treeSitterTreeToDispose) { + treeSitterTreeToDispose.dispose(); + } + } + })); + }); + } + + dispose(): void { + this._disposableStore.dispose(); + for (const tree of this._treeSittersColorizationTrees) { + tree.dispose(); + } + } + + clearCache(): void { + for (const tree of this._treeSittersColorizationTrees) { + tree.dispose(); + } + this._treeSittersColorizationTrees.length = 0; + this._disposableStore.clear(); + this._treeSitterService.clearCache(); + } +} + +registerSingleton(ITreeSitterColorizationService, TreeSitterColorizationService, true); + +// Asynchronous colorization that runs when the process is idle +registerAction2(class extends Action2 { + constructor() { + super({ id: 'toggleAsynchronousTreeSitterColorization', title: 'Toggle Asynchronous Tree-Sitter Colorization', f1: true }); + } + run(accessor: ServicesAccessor) { + const treeSitterTokenizationService = accessor.get(ITreeSitterColorizationService); + treeSitterTokenizationService.clearCache(); + treeSitterTokenizationService.registerTreeSittersForColorization(true); + } +}); + +// Synchronous colorization for testing the performance +registerAction2(class extends Action2 { + constructor() { + super({ id: 'toggleSynchronousTreeSitterColorization', title: 'Toggle Synchronous Tree-Sitter Colorization', f1: true }); + } + run(accessor: ServicesAccessor) { + const treeSitterTokenizationService = accessor.get(ITreeSitterColorizationService); + treeSitterTokenizationService.clearCache(); + treeSitterTokenizationService.registerTreeSittersForColorization(false); + } +}); diff --git a/src/vs/editor/browser/services/treeSitterServices/treeSitterService.test.ts b/src/vs/editor/browser/services/treeSitterServices/treeSitterService.test.ts new file mode 100644 index 0000000000000..c80b9ccc4b239 --- /dev/null +++ b/src/vs/editor/browser/services/treeSitterServices/treeSitterService.test.ts @@ -0,0 +1,107 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { ITreeSitterService, TreeSitterService } from 'vs/editor/browser/services/treeSitterServices/treeSitterService'; +import { LanguageFeatureDebounceService } from 'vs/editor/common/services/languageFeatureDebounce'; +import { LanguageFeaturesService } from 'vs/editor/common/services/languageFeaturesService'; +import { LanguageService } from 'vs/editor/common/services/languageService'; +import { ModelService } from 'vs/editor/common/services/modelService'; +import { ServiceCollection } from 'vs/platform/instantiation/common/serviceCollection'; +import { NullLogService } from 'vs/platform/log/common/log'; +import { UndoRedoService } from 'vs/platform/undoRedo/common/undoRedoService'; +import { Range } from 'vs/editor/common/core/range'; +import { Selection } from 'vs/editor/common/core/selection'; +import { FileService } from 'vs/platform/files/common/fileService'; +import { Schemas } from 'vs/base/common/network'; +import { InstantiationService } from 'vs/platform/instantiation/common/instantiationService'; +import { IModelService } from 'vs/editor/common/services/model'; +import { IFileService } from 'vs/platform/files/common/files'; +/* eslint-disable */ +import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService'; +import { createTextModel } from 'vs/editor/test/common/testTextModel'; +import { TestConfigurationService } from 'vs/platform/configuration/test/common/testConfigurationService'; +import { TestDialogService } from 'vs/platform/dialogs/test/common/testDialogService'; +import { TestNotificationService } from 'vs/platform/notification/test/common/testNotificationService'; +import { TestThemeService } from 'vs/platform/theme/test/common/testThemeService'; +import { TestTextResourcePropertiesService } from 'vs/workbench/test/common/workbenchTestServices'; +import { DiskFileSystemProvider } from 'vs/platform/files/node/diskFileSystemProvider'; +/* eslint-enable*/ + +suite('Testing the Tree-Sitter Service', () => { + + const configService = new TestConfigurationService(); + configService.setUserConfiguration('editor', { 'detectIndentation': false }); + const dialogService = new TestDialogService(); + const notificationService = new TestNotificationService(); + const logService = new NullLogService(); + const nullLogService = new NullLogService(); + const modelService = new ModelService( + configService, + new TestTextResourcePropertiesService(configService), + new TestThemeService(), + nullLogService, + new UndoRedoService(dialogService, notificationService), + new LanguageService(), + new TestLanguageConfigurationService(), + new LanguageFeatureDebounceService(logService), + new LanguageFeaturesService() + ); + + const diskFileSystemProvider = new DiskFileSystemProvider(logService); + const fileService = new FileService(nullLogService); + fileService.registerProvider(Schemas.file, diskFileSystemProvider); + + const treeSitterService = new TreeSitterService(modelService, fileService); + + const servicesCollection = new ServiceCollection( + [IModelService, modelService], + [IFileService, fileService], + [ITreeSitterService, treeSitterService] + ); + + const text = [ + 'function foo() {', + '', + '}', + '/* comment related to TestClass', + ' end of the comment */', + '@classDecorator', + 'class TestClass {', + '// comment related to the function functionOfClass', + 'functionOfClass(){', + 'function function1(){', + '}', + '}}', + 'function bar() { function insideBar() {}', + '}' + ].join('\n'); + + const instantiationService = new InstantiationService(servicesCollection); + const treeSitterServiceInstance = instantiationService.createInstance(TreeSitterService); + + // TODO: fix - test retrieves wrong web-tree-sitter bindings + test.skip('Checking that parse tree not recomputed if already computed', async () => { + + const model = createTextModel(text, 'typescript'); + const tree = await treeSitterServiceInstance.getTreeSitterTree(model); + await Promise.all([tree.parseTreeAndCountCalls(), tree.parseTreeAndCountCalls()]).then((values) => { + console.log('Values returned are : ', values); + }); + model.dispose(); + }); + + // TODO: fix - test retrieves wrong web-tree-sitter bindings + test.skip('Checking that parse tree is recomputed when edit is made', async () => { + const model = createTextModel(text, 'typescript'); + const tree = await treeSitterServiceInstance.getTreeSitterTree(model); + await tree.parseTreeAndCountCalls().then((value: number) => { + console.log('value 1 : ', value); + }); + model.pushEditOperations(null, [{ range: new Range(1, 5, 1, 5), text: '1' }], () => [new Selection(1, 5, 1, 5)]); + await tree.parseTreeAndCountCalls().then((value: number) => { + console.log('value 2 : ', value); + }); + model.dispose(); + }); +}); diff --git a/src/vs/editor/browser/services/treeSitterServices/treeSitterService.ts b/src/vs/editor/browser/services/treeSitterServices/treeSitterService.ts new file mode 100644 index 0000000000000..829338047b307 --- /dev/null +++ b/src/vs/editor/browser/services/treeSitterServices/treeSitterService.ts @@ -0,0 +1,143 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// eslint-disable-next-line local/code-import-patterns +import Parser = require('web-tree-sitter'); +import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; +import { ITextModel } from 'vs/editor/common/model'; +import { URI } from 'vs/base/common/uri'; +import { FileAccess } from 'vs/base/common/network'; +import { TreeSitterTree as TreeSitterTree } from 'vs/editor/browser/services/treeSitterServices/treeSitterTree'; +import { registerSingleton } from 'vs/platform/instantiation/common/extensions'; +import { IModelService } from 'vs/editor/common/services/model'; +import { DisposableStore } from 'vs/base/common/lifecycle'; +import { IFileService } from 'vs/platform/files/common/files'; +import { StopWatch } from 'vs/base/common/stopwatch'; + +export const ITreeSitterService = createDecorator('ITreeSitterService'); + +export interface ITreeSitterService { + readonly _serviceBrand: undefined; + getTreeSitterCaptures(model: ITextModel, queryString: string, asychronous?: boolean, startLine?: number): Promise; + registerLanguage(language: string, uri: string): void; + getTreeSitterTree(model: ITextModel): Promise; + clearCache(): void; +} + +export class TreeSitterService implements ITreeSitterService { + + readonly _serviceBrand: undefined; + private _language: Parser.Language | undefined = undefined; + private _trees: Map = new Map(); + private readonly _store: DisposableStore = new DisposableStore(); + private readonly _fileService: IFileService; + private readonly _modelService: IModelService; + private supportedLanguages = new Map([ + ['typescript', './tree-sitter-typescript.wasm'] + ]); + + constructor( + @IModelService _modelService: IModelService, + @IFileService _fileservice: IFileService + ) { + this._fileService = _fileservice; + this._modelService = _modelService; + this._store.add(_modelService.onModelRemoved((model) => { + if (this._trees.has(model.uri)) { + const treeSitterTree = this._trees.get(model.uri); + this._trees.delete(model.uri); + treeSitterTree!.dispose(); + } + })); + } + + public registerLanguage(language: string, uri: string): void { + this.supportedLanguages.set(language, uri); + } + + public async getTreeSitterCaptures(model: ITextModel, queryString: string, asychronous: boolean = true, startLine?: number): Promise { + if (!this._language) { + return this.fetchLanguage(model.getLanguageId()).then((language) => { + this._language = language; + return this._getTreeSitterCaptures(model, queryString, asychronous, startLine); + }); + } else { + return this._getTreeSitterCaptures(model, queryString, asychronous, startLine); + } + } + + private async _getTreeSitterCaptures(model: ITextModel, queryString: string, asynchronous: boolean = true, startLine?: number): Promise { + if (!this._language) { + throw new Error('Parser language should be defined'); + } + if (!this._trees.has(model.uri)) { + this._trees.set(model.uri, new TreeSitterTree(model, this._language, this._modelService, asynchronous)); + } + + const tree = this._trees.get(model.uri); + const sw = StopWatch.create(true); + const parsedTree = await tree!.parseTree(); + const timeTreeParse = sw.elapsed(); + console.log('Time to parse tree : ', timeTreeParse); + const query = this._language.query(queryString); + sw.reset(); + const captures = query.captures(parsedTree.rootNode, { row: startLine ? startLine : 1, column: 1 } as Parser.Point); + const timeCaptureQueries = sw.elapsed(); + console.log('Time to get the query captures : ', timeCaptureQueries); + query.delete(); + return captures; + } + + public async getTreeSitterTree(model: ITextModel): Promise { + if (!this._language) { + return this.fetchLanguage(model.getLanguageId()).then((language) => { + this._language = language; + return this._getTreeSitterTree(model); + }); + } else { + return this._getTreeSitterTree(model); + } + } + + private _getTreeSitterTree(model: ITextModel): TreeSitterTree { + if (!this._language) { + throw new Error('Parser language should be defined'); + } + if (this._trees.has(model.uri)) { + return this._trees.get(model.uri)!; + } else { + this._trees.set(model.uri, new TreeSitterTree(model, this._language, this._modelService)); + return this._trees.get(model.uri)!; + } + } + + private async fetchLanguage(language: string): Promise { + if (!this.supportedLanguages.has(language)) { + throw new Error('Unsupported language in tree-sitter'); + } + const languageFile = await (this._fileService.readFile(FileAccess.asFileUri(this.supportedLanguages.get(language)!, require))); + return Parser.Language.load(languageFile.value.buffer).then((language: Parser.Language) => { + return new Promise(function (resolve, _reject) { + resolve(language); + }); + }); + } + + dispose(): void { + for (const tree of this._trees.values()) { + tree.dispose(); + } + this._store.dispose(); + } + + clearCache(): void { + for (const tree of this._trees.values()) { + tree.dispose(); + } + this._store.clear(); + this._trees.clear(); + } +} + +registerSingleton(ITreeSitterService, TreeSitterService, true); diff --git a/src/vs/editor/browser/services/treeSitterServices/treeSitterTree.ts b/src/vs/editor/browser/services/treeSitterServices/treeSitterTree.ts new file mode 100644 index 0000000000000..588191595af73 --- /dev/null +++ b/src/vs/editor/browser/services/treeSitterServices/treeSitterTree.ts @@ -0,0 +1,180 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// eslint-disable-next-line local/code-import-patterns +import Parser = require('web-tree-sitter'); +import { runWhenIdle } from 'vs/base/common/async'; +import { ITextModel } from 'vs/editor/common/model'; +import { Position } from 'vs/editor/common/core/position'; +import { Range } from 'vs/editor/common/core/range'; +import { IModelContentChangedEvent } from 'vs/editor/common/textModelEvents'; +import { DisposableStore } from 'vs/base/common/lifecycle'; +import { IModelService } from 'vs/editor/common/services/model'; +import { setTimeout0 } from 'vs/base/common/platform'; + +export class TreeSitterTree { + + private readonly _parser: Parser; + private _tree: Parser.Tree | undefined; + private _edits: Parser.Edit[]; + private _nCallsParseTree: number; + private _nCallsParseAsync: number; + private readonly _modelService: IModelService; + private readonly _store: DisposableStore = new DisposableStore(); + + constructor( + private readonly _model: ITextModel, + _language: Parser.Language, + _modelService: IModelService, + private readonly _asynchronous: boolean = true + ) { + this._modelService = _modelService; + this._parser = new Parser(); + this._parser.setLanguage(_language); + this._edits = []; + this._nCallsParseTree = 0; + this._nCallsParseAsync = 0; + this._store.add(this._model.onDidChangeContent((contentChangeEvent: IModelContentChangedEvent) => { + this.registerTreeEdits(contentChangeEvent); + })); + } + + public registerTreeEdits(contentChangeEvent: IModelContentChangedEvent): void { + for (const change of contentChangeEvent.changes) { + const newEndPositionFromModel = this._model.getPositionAt(change.rangeOffset + change.text.length); + this._edits.push({ + startPosition: { row: change.range.startLineNumber - 1, column: change.range.startColumn - 1 }, + oldEndPosition: { row: change.range.endLineNumber - 1, column: change.range.endColumn - 1 }, + newEndPosition: { row: newEndPositionFromModel.lineNumber - 1, column: newEndPositionFromModel.column - 1 }, + startIndex: change.rangeOffset, + oldEndIndex: change.rangeOffset + change.rangeLength, + newEndIndex: change.rangeOffset + change.text.length + } as Parser.Edit); + } + } + + public async parseTree(): Promise { + this._nCallsParseTree = 0; + return this._parseTree(); + } + + public async parseTreeAndCountCalls(): Promise { + this._nCallsParseTree = 0; + return this._parseTree().then(() => { + return Promise.resolve(this._nCallsParseTree); + }); + } + + private _currentParseOperation: Promise | undefined; + + private async _parseTree(): Promise { + await this._currentParseOperation; + // Case 1: Either there is no tree yet or there are edits to parse + if (!this._tree || this._edits.length !== 0) { + const myParseOperation = this._tryParseSync(); + this._currentParseOperation = myParseOperation; + myParseOperation.then((tree) => { + if (this._currentParseOperation === myParseOperation) { + this._currentParseOperation = undefined; + } + if (this._edits.length !== 0) { + return this._parseTree(); + } + this._nCallsParseTree += 1; + return tree; + }); + this._nCallsParseTree += 1; + return this._currentParseOperation; + } + // Case 2: Else + else { + this._nCallsParseTree += 1; + return this._tree; + } + } + + private async _tryParseSync(): Promise { + if (this._asynchronous) { + this._parser.setTimeoutMicros(10000); + } + const tree = this.updateAndGetTree(); + // Initially synchronous + try { + const result = this._parser.parse( + (startIndex: number, startPoint: Parser.Point | undefined, endIndex: number | undefined) => + this._retrieveTextAtPosition(this._model, startIndex, startPoint, endIndex), + tree + ); + this._tree = result; + return result; + } + // Else if parsing failed, asynchronous + catch (error) { + const model = this._modelService.createModel('', null); + model.setValue(this._model.createSnapshot()); + this._nCallsParseAsync = 0; + return new Promise((resolve, _reject) => { + this._parseAsync(model, tree).then((tree) => { + this._tree = tree; + resolve(tree); + }); + }); + } + } + + private updateAndGetTree(): Parser.Tree | undefined { + if (!this._tree) { + return undefined; + } + for (const edit of this._edits) { + this._tree.edit(edit); + } + this._edits.length = 0; + return this._tree; + } + + private _parseAsync(textModel: ITextModel, tree: Parser.Tree | undefined): Promise { + this._nCallsParseAsync += 1; + return new Promise((resolve, _reject) => { + setTimeout0(async () => { + this._parser.setTimeoutMicros(15 * 1000); + let result: Parser.Tree; + try { + result = this._parser.parse( + (startIndex: number, startPoint: Parser.Point | undefined, endIndex: number | undefined) => + this._retrieveTextAtPosition(textModel, startIndex, startPoint, endIndex), + tree + ); + // Case 1: Either we obtain the result this iteration in which case we resolve + this._tree = result; + console.log('Number of calls to _parseAsync : ', this._nCallsParseAsync); + resolve(result); + } + // Case 2: Here in the catch block treat the case when the parse has failed, then rerun the method + catch (error) { + return this._parseAsync(textModel, tree).then((tree) => { + resolve(tree); + }); + } + }); + }); + } + + private _retrieveTextAtPosition(model: ITextModel, startIndex: number, _startPoint: Parser.Point | undefined, endIndex: number | undefined) { + const startPosition: Position = model.getPositionAt(startIndex); + if (typeof endIndex !== 'number') { + endIndex = startIndex + 5000; + } + const endPosition: Position = model.getPositionAt(endIndex); + return model.getValueInRange(Range.fromPositions(startPosition, endPosition)); + } + + public dispose() { + this._store.dispose(); + this._tree?.delete(); + this._parser.delete(); + this._edits.length = 0; + } +} diff --git a/src/vs/editor/browser/services/treeSitterTrees/colorization/treeSitterColorizationQueries.scm b/src/vs/editor/browser/services/treeSitterTrees/colorization/treeSitterColorizationQueries.scm new file mode 100644 index 0000000000000..9137aa57d4ca2 --- /dev/null +++ b/src/vs/editor/browser/services/treeSitterTrees/colorization/treeSitterColorizationQueries.scm @@ -0,0 +1,132 @@ +(comment) @comment.block.ts + +(string) @string.quoted.double.ts + +(number) @constant.numeric.decimal.ts + +(method_signature name: (property_identifier) @entity.name.function.ts) +(function_declaration name: (identifier) @entity.name.function.ts) +(method_definition name: (property_identifier) @entity.name.function.ts) +(call_expression function: (member_expression property: (property_identifier) @entity.name.function.ts)) +(call_expression function: (identifier) @entity.name.function.ts) +(new_expression constructor: (identifier) @entity.name.function.ts) +(pair key: (property_identifier) @entity.name.function.ts) + +[ + "class" +] @storage.type.class.ts + +[ + "const" + "let" +] @storage.type.ts + +[ + "extends" + "static" + "readonly" + "implements" + "extends" + "declare" + "private" + "public" + "readonly" + "protected" +] @storage.modifier.ts + +[ + "new" +] @keyword.operator.new.ts + +[ + "interface" +] @storage.type.interface.ts + +[ + "type" +] @storage.type.type.ts + +[ + "namespace" +] @storage.type.namespace.ts + +[ + "keyof" +] @keyword.operator.expression.keyof.ts + +[ + "enum" +] @storage.type.enum.ts + +[ + "import" +] @keyword.control.import.ts + +[ + "from" +] @keyword.control.from.ts + +[ + "export" +] @keyword.control.export.ts + +[ + "while" + "for" +] @keyword.control.loop.ts + +[ + "if" + "else" +] @keyword.control.conditional.ts + +[ + "return" +] @keyword.control.flow.ts + + +(member_expression object: (this) @variable.language.this.ts) + +(call_expression function: (super) @variable.language.super.ts) + +(undefined) @constant.language.undefined.ts + +(public_field_definition name: (property_identifier) @variable.object.property.ts) +(property_signature name: (property_identifier) @variable.object.property.ts) + +(assignment_expression left : (member_expression property: (property_identifier) @variable.other.property.ts)) + +(member_expression property: (property_identifier) @variable.other.object.property.ts) + +(binary_expression left: (identifier) @variable.other.object.ts) +(call_expression function: (member_expression object: (identifier) @variable.other.object.ts)) + +(import_specifier name : (identifier) @variable.other.readwrite.alias.ts) +(assignment_expression right: (identifier) @variable.other.readwrite.ts) +(decorator (identifier) @variable.other.readwrite.ts) +(arguments (identifier) @variable.other.readwrite.ts) +(binary_expression right: (identifier) @variable.other.readwrite.ts) +(assignment_expression left: (identifier) @variable.other.readwrite.ts) +(unary_expression argument: (identifier) @variable.other.readwrite.ts) +(variable_declarator name: (identifier) @variable.other.readwrite.ts) +(for_in_statement left: (identifier) @variable.other.readwrite.ts) +(update_expression argument: (identifier) @variable.other.readwrite.ts) +(arrow_function body: (identifier) @variable.other.readwrite.ts) +(shorthand_property_identifier) @variable.other.readwrite.ts +(pair value: (identifier) @variable.other.readwrite.ts) + +(false) @constant.language.boolean.false.ts +(true) @constant.language.boolean.true.ts + +(arrow_function parameter: (identifier) @variable.parameter.ts) +(required_parameter (identifier) @variable.parameter.ts) +(optional_parameter (identifier) @variable.parameter.ts) + +(enum_assignment (property_identifier) @variable.other.enummember.ts) +(enum_body (property_identifier) @variable.other.enummember.ts) + +(predefined_type) @support.type.primitive.ts + +(type_identifier) @entity.name.type.class.ts + +(pair key: (property_identifier) @meta.object-literal.key.ts) diff --git a/src/vs/editor/browser/services/treeSitterTrees/colorization/treeSitterColorizationTree.ts b/src/vs/editor/browser/services/treeSitterTrees/colorization/treeSitterColorizationTree.ts new file mode 100644 index 0000000000000..3f55ee3162f0e --- /dev/null +++ b/src/vs/editor/browser/services/treeSitterTrees/colorization/treeSitterColorizationTree.ts @@ -0,0 +1,188 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// TODO: fix the imports +// eslint-disable-next-line local/code-import-patterns +import Parser = require('web-tree-sitter'); +// eslint-disable-next-line local/code-import-patterns +import { ColorThemeData } from 'vs/workbench/services/themes/common/colorThemeData'; +import { DisposableStore } from 'vs/base/common/lifecycle'; +import { ITextModel } from 'vs/editor/common/model'; +import { IModelContentChangedEvent } from 'vs/editor/common/textModelEvents'; +import { ContiguousMultilineTokens } from 'vs/editor/common/tokens/contiguousMultilineTokens'; +import { FileAccess } from 'vs/base/common/network'; +import { IThemeService } from 'vs/platform/theme/common/themeService'; +import { SemanticTokensProviderStylingConstants } from 'vs/editor/common/services/semanticTokensProviderStyling'; +import { FontStyle, MetadataConsts } from 'vs/editor/common/encodedTokenAttributes'; +import { TokenStyle } from 'vs/platform/theme/common/tokenClassificationRegistry'; +import { ITreeSitterService } from 'vs/editor/browser/services/treeSitterServices/treeSitterService'; +import { IFileService } from 'vs/platform/files/common/files'; +import { StopWatch } from 'vs/base/common/stopwatch'; + +export class TreeSitterColorizationTree { + + public id: string; + private readonly _disposableStore: DisposableStore = new DisposableStore(); + private _contiguousMultilineToken: ContiguousMultilineTokens[]; + private _beginningCaptureIndex: number; + private _startPositionRow: number; + private _endPositionRow: number; + private _newEndPositionRow: number; + private _colorThemeData: ColorThemeData; + private _fileService: IFileService; + + constructor( + private readonly _model: ITextModel, + @ITreeSitterService _treeSitterService: ITreeSitterService, + @IThemeService _themeService: IThemeService, + @IFileService _fileService: IFileService, + _asynchronous: boolean = true + ) { + console.log('Asynchronous? ', _asynchronous); + this.id = _model.id; + this._fileService = _fileService; + this._colorThemeData = _themeService.getColorTheme() as ColorThemeData; + this._contiguousMultilineToken = []; + this._beginningCaptureIndex = 0; + this._startPositionRow = 0; + this._endPositionRow = this._model.getLineCount() - 1; + this._newEndPositionRow = this._model.getLineCount() - 1; + + this._fetchQueries().then((query) => { + _treeSitterService.getTreeSitterCaptures(this._model, query, _asynchronous).then((queryCaptures) => { + if (!queryCaptures) { + return; + } + const sw = StopWatch.create(true); + this.setTokensUsingQueryCaptures(queryCaptures).then(() => { + console.log('Time to set tokens : ', sw.elapsed()); + this._disposableStore.add(this._model.onDidChangeContent((contentChangeEvent: IModelContentChangedEvent) => { + this.updateRowIndices(contentChangeEvent); + _treeSitterService.getTreeSitterCaptures(this._model, query, _asynchronous, this._startPositionRow).then((queryCaptures) => { + if (!queryCaptures) { + return; + } + this.setTokensUsingQueryCaptures(queryCaptures); + }); + })); + }); + }); + }); + } + + private async _fetchQueries(): Promise { + const query = await this._fileService.readFile(FileAccess.asFileUri(`./treeSitterColorizationQueries.scm`, require)); + return Promise.resolve(query.value.toString()); + } + + public setTokensUsingQueryCaptures(queryCaptures: Parser.QueryCapture[]): Promise { + this._contiguousMultilineToken.splice(this._startPositionRow, this._model.getLineCount() - this._startPositionRow + 1); + // Case 1: code was removed + if (this._newEndPositionRow < this._endPositionRow) { + this._contiguousMultilineToken.map(token => { + if (token.startLineNumber >= this._endPositionRow + 2) { + token.startLineNumber = token.startLineNumber - (this._endPositionRow - this._startPositionRow); + } + }); + } + // Case 2: code was added + else if (this._newEndPositionRow > this._endPositionRow) { + this._contiguousMultilineToken.map(token => { + if (token.startLineNumber >= this._endPositionRow + 2) { + token.startLineNumber = token.startLineNumber + (this._newEndPositionRow - this._startPositionRow); + } + }); + } + return new Promise((resolve, _reject) => { + return this.setTokensWithThemeData(queryCaptures, resolve); + }); + } + + private setTokensWithThemeData(queryCaptures: Parser.QueryCapture[], resolve: () => void): void { + let newBeginningIndexFound = true; + const numberCaptures = queryCaptures.length; + let beginningCaptureIndex = this._beginningCaptureIndex; + + for (let i = this._startPositionRow; i <= this._model.getLineCount() - 1; i++) { + const contiguousMultilineTokensArray: number[] = []; + let j = beginningCaptureIndex; + + while (j < numberCaptures && queryCaptures[j].node.startPosition.row <= i) { + if (i >= queryCaptures[j].node.startPosition.row && i <= queryCaptures[j].node.endPosition.row) { + if (!newBeginningIndexFound) { + newBeginningIndexFound = true; + beginningCaptureIndex = j; + } + contiguousMultilineTokensArray.push(queryCaptures[j].node.endPosition.column, this.findMetadata(j, queryCaptures)); + } + j++; + } + newBeginningIndexFound = false; + this._contiguousMultilineToken.splice(i, 0, new ContiguousMultilineTokens(i + 1, [new Uint32Array(contiguousMultilineTokensArray)])); + this._beginningCaptureIndex = beginningCaptureIndex; + this._startPositionRow = i + 1; + } + this._model.tokenization.setTokens(this._contiguousMultilineToken); + resolve(); + } + + private findMetadata(index: number, queryCaptures: Parser.QueryCapture[]): number { + const tokenStyle: TokenStyle | undefined = this._colorThemeData.resolveScopes([[queryCaptures![index].name]], {}); + let metadata: number; + if (typeof tokenStyle === 'undefined') { + metadata = SemanticTokensProviderStylingConstants.NO_STYLING; + } else { + metadata = 0; + if (typeof tokenStyle.italic !== 'undefined') { + const italicBit = (tokenStyle.italic ? FontStyle.Italic : 0) << MetadataConsts.FONT_STYLE_OFFSET; + metadata |= italicBit | MetadataConsts.SEMANTIC_USE_ITALIC; + } + if (typeof tokenStyle.bold !== 'undefined') { + const boldBit = (tokenStyle.bold ? FontStyle.Bold : 0) << MetadataConsts.FONT_STYLE_OFFSET; + metadata |= boldBit | MetadataConsts.SEMANTIC_USE_BOLD; + } + if (typeof tokenStyle.underline !== 'undefined') { + const underlineBit = (tokenStyle.underline ? FontStyle.Underline : 0) << MetadataConsts.FONT_STYLE_OFFSET; + metadata |= underlineBit | MetadataConsts.SEMANTIC_USE_UNDERLINE; + } + if (typeof tokenStyle.strikethrough !== 'undefined') { + const strikethroughBit = (tokenStyle.strikethrough ? FontStyle.Strikethrough : 0) << MetadataConsts.FONT_STYLE_OFFSET; + metadata |= strikethroughBit | MetadataConsts.SEMANTIC_USE_STRIKETHROUGH; + } + if (tokenStyle.foreground) { + const tokenStyleForeground = this._colorThemeData.getTokenColorIndex().get(tokenStyle?.foreground); + const foregroundBits = tokenStyleForeground << MetadataConsts.FOREGROUND_OFFSET; + metadata |= foregroundBits | MetadataConsts.SEMANTIC_USE_FOREGROUND; + } + if (metadata === 0) { + metadata = SemanticTokensProviderStylingConstants.NO_STYLING; + } + } + return metadata; + } + + private updateRowIndices(e: IModelContentChangedEvent): void { + this._startPositionRow = Infinity; + this._endPositionRow = -Infinity; + this._newEndPositionRow = -Infinity; + for (const change of e.changes) { + const newEndPositionFromModel = this._model.getPositionAt(change.rangeOffset + change.text.length); + if (change.range.startLineNumber - 1 < this._startPositionRow) { + this._startPositionRow = change.range.startLineNumber - 1; + this._beginningCaptureIndex = 0; + } + if (change.range.endLineNumber - 1 > this._endPositionRow) { + this._endPositionRow = change.range.endLineNumber - 1; + } + if (newEndPositionFromModel.lineNumber - 1 > this._newEndPositionRow) { + this._newEndPositionRow = newEndPositionFromModel.lineNumber - 1; + } + } + } + + public dispose() { + this._disposableStore.clear(); + this._contiguousMultilineToken.length = 0; + } +} diff --git a/src/vs/editor/common/tokens/contiguousMultilineTokens.ts b/src/vs/editor/common/tokens/contiguousMultilineTokens.ts index a8860866aef84..d611e461e4bae 100644 --- a/src/vs/editor/common/tokens/contiguousMultilineTokens.ts +++ b/src/vs/editor/common/tokens/contiguousMultilineTokens.ts @@ -52,6 +52,13 @@ export class ContiguousMultilineTokens { return this._startLineNumber; } + /** + * Set the start line number of the token + */ + public set startLineNumber(line) { + this._startLineNumber = line; + } + /** * (Inclusive) end line number for these tokens. */ diff --git a/src/vs/editor/editor.all.ts b/src/vs/editor/editor.all.ts index aa32f46a8a9ab..556798dbaccc3 100644 --- a/src/vs/editor/editor.all.ts +++ b/src/vs/editor/editor.all.ts @@ -61,3 +61,5 @@ import 'vs/editor/contrib/readOnlyMessage/browser/contribution'; import 'vs/editor/common/standaloneStrings'; import 'vs/base/browser/ui/codicons/codiconStyles'; // The codicons are defined here and must be loaded + +import 'vs/editor/browser/services/treeSitterServices/treeSitterColorizationService'; diff --git a/src/vs/workbench/services/themes/common/colorThemeData.ts b/src/vs/workbench/services/themes/common/colorThemeData.ts index 5ccb9e5f93d09..8fa8b27ffab3d 100644 --- a/src/vs/workbench/services/themes/common/colorThemeData.ts +++ b/src/vs/workbench/services/themes/common/colorThemeData.ts @@ -236,7 +236,7 @@ export class ColorThemeData implements IWorkbenchColorTheme { return undefined; } - private getTokenColorIndex(): TokenColorIndex { + public getTokenColorIndex(): TokenColorIndex { // collect all colors that tokens can have if (!this.tokenColorIndex) { const index = new TokenColorIndex(); diff --git a/yarn.lock b/yarn.lock index 879f96718365a..62ddc36d84304 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10658,6 +10658,18 @@ tree-kill@^1.2.2: resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.2.tgz#4ca09a9092c88b73a7cdc5e8a01b507b0790a0cc" integrity sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A== +tree-sitter-cli@^0.20.6: + version "0.20.6" + resolved "https://registry.yarnpkg.com/tree-sitter-cli/-/tree-sitter-cli-0.20.6.tgz#2a7202190d7bd64e112b451f94573dbe40a04f04" + integrity sha512-tjbAeuGSMhco/EnsThjWkQbDIYMDmdkWsTPsa/NJAW7bjaki9P7oM9TkLxfdlnm4LXd1wR5wVSM2/RTLtZbm6A== + +tree-sitter-typescript@^0.20.1: + version "0.20.1" + resolved "https://registry.yarnpkg.com/tree-sitter-typescript/-/tree-sitter-typescript-0.20.1.tgz#6b338a1414f5ed13cc39e60275ddeaa0f25870a9" + integrity sha512-wqpnhdVYX26ATNXeZtprib4+mF2GlYQB1cjRPibYGxDRiugx5OfjWwLE4qPPxEGdp2ZLSmZVesGUjLWzfKo6rA== + dependencies: + nan "^2.14.0" + ts-loader@^9.2.7: version "9.2.7" resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-9.2.7.tgz#948654099ca96992b62ec47bd9cee5632006e101" @@ -11297,6 +11309,11 @@ watchpack@^2.2.0: glob-to-regexp "^0.4.1" graceful-fs "^4.1.2" +web-tree-sitter@^0.20.7: + version "0.20.7" + resolved "https://registry.yarnpkg.com/web-tree-sitter/-/web-tree-sitter-0.20.7.tgz#b0ddb78e8244221a3100f432c7e162516cd9cd09" + integrity sha512-flC9JJmTII9uAeeYpWF8hxDJ7bfY+leldQryetll8Nv4WgI+MXc6h7TiyAZASWl9uC9TvmfdgOjZn1DAQecb3A== + webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"