[Draft] - Tree Sitter Service (POC)

extensions/typescript-basics/package.json

@@ -53,6 +53,12 @@
         ]
       }
     ],
+    "tsGrammars": [
+      {
+        "language": "typescript",
+        "path": "./tree-sitter-typescript.wasm"
+      }
+    ],
     "grammars": [
       {
         "language": "typescript",

package.json

@@ -91,7 +91,8 @@
     "xterm-addon-webgl": "0.12.0-beta.27",
     "xterm-headless": "4.19.0-beta.21",
     "yauzl": "^2.9.2",
-    "yazl": "^2.4.3"
+    "yazl": "^2.4.3",
+    "web-tree-sitter": "^0.20.5"
   },
   "devDependencies": {
     "7zip": "0.0.6",

src/vs/workbench/api/browser/extensionHost.contribution.ts

@@ -15,6 +15,7 @@ import { IconExtensionPoint } from 'vs/workbench/services/themes/common/iconExte
 import { TokenClassificationExtensionPoints } from 'vs/workbench/services/themes/common/tokenClassificationExtensionPoint';
 import { LanguageConfigurationFileHandler } from 'vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint';
 
+
 // --- mainThread participants
 import './mainThreadBulkEdits';
 import './mainThreadCodeInsets';

src/vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint.ts

@@ -19,6 +19,7 @@ import { getParseErrorMessage } from 'vs/base/common/jsonErrorMessages';
 import { IExtensionResourceLoaderService } from 'vs/workbench/services/extensionResourceLoader/common/extensionResourceLoader';
 import { hash } from 'vs/base/common/hash';
 import { Disposable } from 'vs/base/common/lifecycle';
+import { ITreeSitterService } from 'vs/workbench/services/treeSitter/common/TSService';
 
 interface IRegExp {
 	pattern: string;
@@ -87,6 +88,7 @@ export class LanguageConfigurationFileHandler extends Disposable {
 
 	constructor(
 		@ITextMateService textMateService: ITextMateService,
+		@ITreeSitterService treeSitterService: ITreeSitterService,
 		@ILanguageService private readonly _languageService: ILanguageService,
 		@IExtensionResourceLoaderService private readonly _extensionResourceLoaderService: IExtensionResourceLoaderService,
 		@IExtensionService private readonly _extensionService: IExtensionService,
@@ -109,6 +111,9 @@ export class LanguageConfigurationFileHandler extends Disposable {
 		this._register(textMateService.onDidEncounterLanguage((languageId) => {
 			this._loadConfigurationsForMode(languageId);
 		}));
+		// this._register(treeSitterService.onDidEncounterLanguage((languageId) => {
+		// 	this._loadConfigurationsForMode(languageId);
+		// }));
 	}
 
 	private async _loadConfigurationsForMode(languageId: string): Promise<void> {

src/vs/workbench/services/treeSitter/common/TSGrammars.ts

@@ -0,0 +1,38 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as nls from 'vs/nls';
+import { ExtensionsRegistry, IExtensionPoint } from 'vs/workbench/services/extensions/common/extensionsRegistry';
+import { languagesExtPoint } from 'vs/workbench/services/language/common/languageService';
+
+export interface ITMSyntaxExtensionPoint {
+	language: string;
+	path: string;
+}
+
+export const grammarsExtPoint: IExtensionPoint<ITMSyntaxExtensionPoint[]> = ExtensionsRegistry.registerExtensionPoint<ITMSyntaxExtensionPoint[]>({
+	extensionPoint: 'tsGrammars',
+	deps: [languagesExtPoint],
+	jsonSchema: {
+		description: nls.localize('vscode.extension.contributes.TSGrammars', 'Contributes Tree Sitter grammars.'),
+		type: 'array',
+		defaultSnippets: [{ body: [{ language: '${1:id}', path: './syntaxes/${3:id}.wasm.' }] }],
+		items: {
+			type: 'object',
+			defaultSnippets: [{ body: { language: '${1:id}', scopeName: 'source.${2:id}', path: './syntaxes/${3:id}.tmLanguage.' } }],
+			properties: {
+				language: {
+					description: nls.localize('vscode.extension.contributes.grammars.language', 'Language identifier for which this syntax is contributed to.'),
+					type: 'string'
+				},
+				path: {
+					description: nls.localize('vscode.extension.contributes.grammars.path', 'Path of the tmLanguage file. The path is relative to the extension folder and typically starts with \'./syntaxes/\'.'),
+					type: 'string'
+				},
+			},
+			required: ['language', 'path']
+		}
+	}
+});

src/vs/workbench/services/treeSitter/common/TSService.ts

@@ -0,0 +1,101 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
+import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
+import { grammarsExtPoint } from 'vs/workbench/services/treeSitter/common/TSGrammars';
+import { Disposable, dispose, IDisposable } from 'vs/base/common/lifecycle';
+import { Emitter, Event } from 'vs/base/common/event';
+import { joinPath } from 'vs/base/common/resources';
+import { ILanguageService } from 'vs/editor/common/languages/language';
+import { URI } from 'vs/base/common/uri';
+import { ITokenizationSupport, ITokenizationSupportFactory, TokenizationRegistry } from 'vs/editor/common/languages';
+import { TSTokenization } from 'vs/workbench/services/treeSitter/common/TSTokenization';
+
+
+
+export interface IValidTSGrammarDefinition {
+	location: URI;
+	language?: string;
+}
+
+export interface ITreeSitterService {
+	readonly _serviceBrand: undefined;
+
+	onDidEncounterLanguage: Event<string>;
+}
+
+export const ITreeSitterService = createDecorator<ITreeSitterService>('treeSitterService');
+
+class TSService extends Disposable {
+	public _serviceBrand: undefined;
+	private _grammarDefinitions: Map<string, IValidTSGrammarDefinition | null>;
+	private readonly _createdModes: string[];
+	private _tokenizersRegistrations: IDisposable[];
+
+	private readonly _onDidEncounterLanguage: Emitter<string> = this._register(new Emitter<string>());
+	public readonly onDidEncounterLanguage: Event<string> = this._onDidEncounterLanguage.event;
+
+	constructor(
+		@ILanguageService protected readonly _languageService: ILanguageService,
+	) {
+		super();
+		this._grammarDefinitions = new Map();
+		this._tokenizersRegistrations = [];
+		this._createdModes = [];
+		grammarsExtPoint.setHandler((extensions) => {
+			this._tokenizersRegistrations = dispose(this._tokenizersRegistrations);
+			for (const extension of extensions) {
+				const grammars = extension.value;
+				for (const grammar of grammars) {
+					const grammarLocation = joinPath(extension.description.extensionLocation, grammar.path);
+
+					let validLanguageId: string | null = null;
+					if (grammar.language && this._languageService.isRegisteredLanguageId(grammar.language)) {
+						validLanguageId = grammar.language;
+					}
+
+					if (validLanguageId) {
+						this._tokenizersRegistrations.push(TokenizationRegistry.registerFactory(validLanguageId, this._createFactory(validLanguageId)));
+
+						this._grammarDefinitions?.set(validLanguageId, {
+							location: grammarLocation,
+							language: validLanguageId ?? undefined
+						});
+					}
+				}
+			}
+
+			for (const createMode of this._createdModes) {
+				TokenizationRegistry.getOrCreate(createMode);
+			}
+		});
+
+		this._languageService.onDidEncounterLanguage((languageId) => {
+			this._createdModes.push(languageId);
+		});
+	}
+
+	private _createFactory(languageId: string): ITokenizationSupportFactory {
+		return {
+			createTokenizationSupport: async (): Promise<ITokenizationSupport | null> => {
+				if (!this._languageService.isRegisteredLanguageId(languageId)) {
+					return null;
+				}
+
+				const grammar = this._grammarDefinitions.get(languageId);
+				if (!grammar) {
+					return null;
+				}
+
+				const tsTokenizationInstance = new TSTokenization(grammar);
+				await tsTokenizationInstance.init();
+				return tsTokenizationInstance;
+			}
+		};
+	}
+
+}
+
+registerSingleton(ITreeSitterService, TSService);

src/vs/workbench/services/treeSitter/common/TSTokenization.ts

@@ -0,0 +1,42 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import { Disposable } from 'vs/base/common/lifecycle';
+import { EncodedTokenizationResult, IState, ITokenizationSupport, TokenizationResult } from 'vs/editor/common/languages';
+import { IValidTSGrammarDefinition } from 'vs/workbench/services/treeSitter/common/TSService';
+import * as Parser from 'web-tree-sitter';
+
+export class TSTokenization extends Disposable implements ITokenizationSupport {
+	private parser: Parser | null;
+	private _grammar: IValidTSGrammarDefinition;
+
+	constructor(grammar: IValidTSGrammarDefinition) {
+		super();
+		this.parser = null;
+		this._grammar = grammar;
+	}
+
+	public async init() {
+		try {
+			await Parser.init();
+		} catch (e) {
+			console.log(e);
+		}
+		// this.parser = new Parser();
+		// const language = await Parser.Language.load(this._grammar.location.path);
+		// this.parser.setLanguage(language);
+	}
+
+	public getInitialState(): IState {
+		throw new Error('Not supported!');
+	}
+
+	public tokenize(line: string, hasEOL: boolean, state: IState): TokenizationResult {
+		throw new Error('Not supported!');
+	}
+
+	public tokenizeEncoded(line: string, hasEOL: boolean): EncodedTokenizationResult {
+		throw new Error('Not supported!');
+	}
+}

src/vs/workbench/workbench.sandbox.main.ts

@@ -38,6 +38,7 @@ import 'vs/workbench/services/textfile/electron-sandbox/nativeTextFileService';
 import 'vs/workbench/services/dialogs/electron-sandbox/fileDialogService';
 import 'vs/workbench/services/workspaces/electron-sandbox/workspacesService';
 import 'vs/workbench/services/textMate/browser/nativeTextMateService';
+import 'vs/workbench/services/treeSitter/common/TSService';
 import 'vs/workbench/services/menubar/electron-sandbox/menubarService';
 import 'vs/workbench/services/issue/electron-sandbox/issueService';
 import 'vs/workbench/services/update/electron-sandbox/updateService';

yarn.lock

@@ -11863,6 +11863,11 @@ watchpack@^2.2.0:
     glob-to-regexp "^0.4.1"
     graceful-fs "^4.1.2"
 
+web-tree-sitter@^0.20.5:
+  version "0.20.5"
+  resolved "https://registry.yarnpkg.com/web-tree-sitter/-/web-tree-sitter-0.20.5.tgz#62c8ea29d94f6ef6f03ce9c68c860df011ee26c7"
+  integrity sha512-mpXlqIeEBE5Q71cnBnt8w6XKhIiKmllPECqsIFBtMvzcfCxA8+614iyMJXBCQo95Vs3y1zORLqiLJn25pYZ4Tw==
+
 webidl-conversions@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"