Use tree-sitter for syntax coloring

package.json

@@ -34,7 +34,8 @@
     "test": "node ./out/test/runTest.js",
     "lint": "node ./node_modules/tslint/bin/tslint --project tsconfig.json",
     "fix-lint": "node ./node_modules/tslint/bin/tslint --fix --project tsconfig.json",
-    "unit-test": "node ./node_modules/mocha/bin/_mocha -u tdd --timeout 5000 --colors ./out/test/unit"
+    "unit-test": "node ./node_modules/mocha/bin/_mocha -u tdd --timeout 5000 --colors ./out/test/unit",
+    "gen-parser": "tree-sitter build-wasm ./node_modules/tree-sitter-go"
   },
   "extensionDependencies": [],
   "dependencies": {
@@ -46,7 +47,9 @@
     "vscode-debugprotocol": "^1.36.0",
     "vscode-extension-telemetry": "^0.1.2",
     "vscode-languageclient": "~5.2.1",
-    "web-request": "^1.0.7"
+    "vscode-tree-sitter": "^0.1.24",
+    "web-request": "^1.0.7",
+    "web-tree-sitter": "^0.15.9"
   },
   "devDependencies": {
     "@types/fs-extra": "^8.0.0",
@@ -56,6 +59,8 @@
     "@types/semver": "^6.0.1",
     "@types/vscode": "^1.25.0",
     "fs-extra": "^8.1.0",
+    "tree-sitter-cli": "^0.15.9",
+    "tree-sitter-go": "^0.15.0",
     "glob": "^7.1.4",
     "mocha": "^6.2.0",
     "tslint": "^5.19.0",
@@ -104,6 +109,11 @@
       }
     ],
     "grammars": [
+      {
+        "language": "go",
+        "scopeName": "source.go",
+        "path": "./syntaxes/go.tmGrammar.json"
+      },
       {
         "language": "go.mod",
         "scopeName": "go.mod",

parsers/README.md

@@ -0,0 +1,3 @@
+`tree-sitter-go.wasm` is an incremental parser for Go using the [Tree Sitter](http://tree-sitter.github.io/tree-sitter/) framework. It's generated by `npm run-script gen-parser`. 
+
+It's more convenient to check the .wasm file into source than to run `gen-parser` as part of the build, because `gen-parser` requires a C++ toolchain on the host. If you want to regenerate `tree-sitter-go.wasm`, I recommend using an Ubuntu linux host.

src/goMain.ts

@@ -35,6 +35,9 @@ import { buildCode } from './goBuild';
 import { installCurrentPackage } from './goInstall';
 import { check, removeTestStatus, notifyIfGeneratedFile } from './goCheck';
 import { GO111MODULE } from './goModules';
+import { activate as activateTreeSitter } from 'vscode-tree-sitter';
+import { color } from './treeSitter';
+import * as path from 'path';
 
 export let buildDiagnosticCollection: vscode.DiagnosticCollection;
 export let lintDiagnosticCollection: vscode.DiagnosticCollection;
@@ -359,6 +362,32 @@ export function activate(ctx: vscode.ExtensionContext): void {
 	});
 
 	sendTelemetryEventForConfig(vscode.workspace.getConfiguration('go', vscode.window.activeTextEditor ? vscode.window.activeTextEditor.document.uri : null));
+	// Parse .go files incrementally using tree-sitter
+	const parserPath = path.join(ctx.extensionPath, 'parsers', 'tree-sitter-go.wasm');
+	activateTreeSitter(ctx, {'go': {wasm: parserPath}}).then(colorAll);
+	function colorAll() {
+		for (const editor of vscode.window.visibleTextEditors) {
+			color(editor);
+		}
+	}
+	function colorEdited(evt: vscode.TextDocumentChangeEvent) {
+		for (const editor of vscode.window.visibleTextEditors) {
+			if (editor.document.uri.toString() === evt.document.uri.toString()) {
+				color(editor);
+			}
+		}
+	}
+	function onChangeConfiguration(event: vscode.ConfigurationChangeEvent) {
+		const colorizationNeedsReload: boolean = event.affectsConfiguration('workbench.colorTheme')
+			|| event.affectsConfiguration('editor.tokenColorCustomizations');
+		if (colorizationNeedsReload) {
+			colorAll();
+		}
+	}
+	ctx.subscriptions.push(vscode.workspace.onDidChangeConfiguration(onChangeConfiguration));
+	ctx.subscriptions.push(vscode.window.onDidChangeVisibleTextEditors(colorAll));
+	ctx.subscriptions.push(vscode.window.onDidChangeTextEditorVisibleRanges(change => color(change.textEditor)));
+	ctx.subscriptions.push(vscode.workspace.onDidChangeTextDocument(colorEdited));
 }
 
 export function deactivate() {

src/treeSitter.ts

@@ -0,0 +1,32 @@
+import vscode = require('vscode');
+import {tree, decoration} from 'vscode-tree-sitter';
+import {colorGo, Range} from './treeSitterColor';
+
+export function color(editor: vscode.TextEditor) {
+	try {
+		if (editor.document.languageId !== 'go') return;
+		const visibleRanges = editor.visibleRanges.map(range => {
+			const start = range.start.line;
+			const end = range.end.line;
+			return {start, end};
+		});
+		const t = tree(editor.document.uri);
+		if (t == null) {
+			console.warn(editor.document.uri.path, 'has not been parsed');
+			return;
+		}
+		const colors = colorGo(t, visibleRanges);
+		for (const scope of Object.keys(colors)) {
+			const dec = decoration(scope);
+			if (!dec) continue;
+			const ranges = colors[scope]!.map(range);
+			editor.setDecorations(dec, ranges);
+		}
+	} catch (e) {
+		console.error(e);
+	}
+}
+
+function range(x: Range): vscode.Range {
+	return new vscode.Range(x.start.row, x.start.column, x.end.row, x.end.column);
+}