clean up the demo now that it is working as intended!

bolinfest · May 1, 2020 · ca4e82b · ca4e82b
1 parent d305813
commit ca4e82b
Show file tree

Hide file tree

Showing 6 changed files with 69 additions and 197 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # monaco-tm
 
-This attempts to get TextMate grammars working in standalne Monaco by leveraging
+This gets TextMate grammars working in standalone Monaco by leveraging
 `vscode-oniguruma` and `vscode-textmate`. For more context, see:
 https://github.com/microsoft/monaco-editor/issues/1915.
 
@@ -10,18 +10,5 @@ https://github.com/microsoft/monaco-editor/issues/1915.
 - `yarn demo`
 - open http://localhost:8084/
 
-## Status
-
-Currently, I am trying to use this to verify I can get the Hack grammar working
-in standalone Monaco. Unfortunately, it is not working yet.
-
-To try other languages, play with `options` at the bottom of `app.ts`.
-
-I also took a brief look at:
-
-- https://github.com/NeekSandhu/monaco-textmate
-- https://github.com/NeekSandhu/monaco-editor-textmate
-
-I believe that demo reimplements some of what `vscode-textmate` provides.
-I would prefer to have something as close to the VS Code implementation as
-possible.
+This shows off the Hack grammar working by default, as it is a language for
+which a TextMate grammar exists, but no Monarch grammar.
diff --git a/package.json b/package.json
@@ -8,10 +8,7 @@
   },
   "dependencies": {
     "monaco-editor": "0.20.0",
-    "monaco-editor-textmate": "2.2.1",
-    "monaco-textmate": "3.0.1",
     "nullthrows": "1.1.1",
-    "onigasm": "2.2.4",
     "vscode-oniguruma": "1.3.0",
     "vscode-textmate": "5.1.1"
   },

diff --git a/src/app.ts b/src/app.ts
@@ -1,4 +1,4 @@
-import type {GrammarStore} from './index';
+import type {SupportedLanguage} from './examples';
 
 // Recall we are using MonacoWebpackPlugin. According to the
 // monaco-editor-webpack-plugin docs, we must use:
@@ -13,66 +13,13 @@ import type {GrammarStore} from './index';
 import * as monaco from 'monaco-editor/esm/vs/editor/editor.api';
 import nullthrows from 'nullthrows';
 import {createGrammarStore} from './index';
-import {INITIAL} from 'vscode-textmate';
-
-// From monaco-editor-textmate README.
-import {Registry} from 'monaco-textmate';
-import {wireTmGrammars} from 'monaco-editor-textmate';
-import {loadWASM} from 'onigasm';
+import {getSampleCodeForLanguage} from './examples';
 
 type GrammarConfiguration = {language: string; scopeName: string; url: string};
 
-const useEncodedTokens = true;
-main(useEncodedTokens, 'hack');
-
-const testOptions = {
-  hack: `<?hh // strict
-
-type SomeShape = shape(
-  'frame_type' => FrameType,
-  ?'commit_id' => ?string,
-);
+main('hack');
 
-abstract final class ClassDefWithLotsOfKeywords {
-  const int BIG_NUMBER = 30000000;
-
-  public static async function genRender(): Awaitable<Stuff> {
-    return <xhp:div>
-      hello world
-    </xhp:div>;
-  }
-}
-`,
-  html: `<!DOCTYPE HTML>
-<html>
-<head>
-</head>
-<body>
-</body>
-</html>
-`,
-  javascript: `\
-const React = require('react');
-
-function create() {
-  return (
-    <div>
-      hello world
-    </div>
-  );
-}
-`,
-  python: `\
-import foo
-
-async def bar(): string:
-  f = await foo()
-  f_string = f"Hooray {f}! format strings are not supported in current Monarch grammar"
-  return foo_string
-`,
-};
-
-async function main(useEncodedTokens: boolean, testLanguage: keyof typeof testOptions) {
+async function main(language: SupportedLanguage) {
   // Note that Hack lists text.html.basic as an embedded grammar, so we must
   // provide that grammar (and all of its transitive deps) as well.
   //
@@ -96,125 +43,27 @@ async function main(useEncodedTokens: boolean, testLanguage: keyof typeof testOp
     });
   }
 
-  if (useEncodedTokens) {
-    await tryEncodedTokensProvider(grammarConfigurations);
-  } else {
-    await tryMonacoEditorTextMate(grammarConfigurations);
-  }
-
-  const value = testOptions[testLanguage];
+  await registerEncodedTokensProviders(grammarConfigurations);
 
-  const theme = 'hackTheme';
-  defineTheme(theme);
+  const value = getSampleCodeForLanguage(language);
   monaco.editor.create(nullthrows(document.getElementById('container')), {
     value,
-    language: testLanguage,
-    theme,
+    language,
+    theme: 'vs', // 'vs' or 'vs-dark' should both work here
     minimap: {
       enabled: false,
     },
   });
 }
 
-async function tryCodeOnVSCodeTextMateReadme(grammarStore: GrammarStore) {
-  const grammar = nullthrows(await grammarStore.getGrammar('source.hack'));
-  const text = `<?hh // strict
-
-class Example {
-}
-`.split('\n');
-  let ruleStack = INITIAL;
-  for (let i = 0; i < text.length; i++) {
-    const line = text[i];
-    const lineTokens = grammar.tokenizeLine(line, ruleStack);
-    console.log(`\nTokenizing line: ${line}`);
-    for (let j = 0; j < lineTokens.tokens.length; j++) {
-      const token = lineTokens.tokens[j];
-      console.log(
-        ` - token from ${token.startIndex} to ${token.endIndex} ` +
-          `(${line.substring(token.startIndex, token.endIndex)}) ` +
-          `with scopes ${token.scopes.join(', ')}`,
-      );
-    }
-    ruleStack = lineTokens.ruleStack;
-  }
-}
-
-async function tryEncodedTokensProvider(grammarConfigurations: GrammarConfiguration[]) {
+async function registerEncodedTokensProviders(grammarConfigurations: GrammarConfiguration[]) {
   const scopeNameToTextMateGrammarURL: Map<string, string> = new Map(
     grammarConfigurations.map(({scopeName, url}) => [scopeName, url]),
   );
   const grammarStore = await createGrammarStore(scopeNameToTextMateGrammarURL);
 
   for (const {language, scopeName} of grammarConfigurations) {
-    // const tokensProvider = await grammarStore.createTokensProvider(scopeName);
     const tokensProvider = await grammarStore.createEncodedTokensProvider(scopeName);
     monaco.languages.setTokensProvider(language, tokensProvider);
   }
-
-  // Although the web demo doesn't work, this seems to have sensible output.
-  await tryCodeOnVSCodeTextMateReadme(grammarStore);
-}
-
-function defineTheme(name: string): void {
-  // This code is ported from this playground:
-  // https://microsoft.github.io/monaco-editor/playground.html#customizing-the-appearence-tokens-and-colors
-  // It seems to work there, so we ignore these errors.
-  // @ts-ignore
-  monaco.editor.defineTheme(name, {
-    base: 'vs-dark',
-    inherit: true,
-    rules: [
-      {token: 'constant.numeric', foreground: 'B5CEA8'},
-      {token: 'constant.other', foreground: 'D4D4D4'},
-      {token: 'keyword.operator.comparison', foreground: 'D4D4D4'},
-      {token: 'entity.name.function', foreground: 'DCDCAA'},
-      {token: 'entity.name.tag', foreground: '569CD6'},
-      {token: 'entity.name.type', foreground: '4EC9B0'},
-      {token: 'storage.modifier', foreground: '569CD6'},
-      {token: 'storage.type', foreground: '569CD6'},
-      {token: 'support.class', foreground: '4EC9B0'},
-
-      // Multiple comment defs necessary?
-      {token: 'comment', foreground: '6A9955'},
-      {token: 'punctuation.definition.comment', foreground: '6A9955'},
-
-      // Multiple string defs necessary?
-      {token: 'string', foreground: 'CE9178'},
-      {token: 'string.quoted.single', foreground: 'CE9178'},
-      {token: 'meta.string-contents.quoted.single', foreground: 'CE9178'},
-      {token: 'punctuation.definition.string', foreground: 'CE9178'},
-
-      // Multiple variable defs necessary?
-      {token: 'punctuation.definition.variable', foreground: '9CDCFE'},
-      {token: 'variable', foreground: '9CDCFE'},
-    ],
-  });
-}
-
-// Adapted from the README for monaco-editor-textmate.
-async function tryMonacoEditorTextMate(grammarConfigurations: GrammarConfiguration[]) {
-  await loadWASM('/node_modules/onigasm/lib/onigasm.wasm');
-
-  const registry = new Registry({
-    getGrammarDefinition: async (scopeName) => {
-      const config = grammarConfigurations.find((config) => config.scopeName === scopeName);
-      if (config == null) {
-        throw Error(`no URL for ${scopeName}`);
-      }
-
-      const {url} = config;
-      const format = url.endsWith('.json') ? 'json' : 'plist';
-      return {
-        format,
-        content: await (await fetch(url)).text(),
-      };
-    },
-  });
-
-  const grammars = new Map(
-    grammarConfigurations.map(({language, scopeName}) => [language, scopeName]),
-  );
-
-  await wireTmGrammars(monaco, registry, grammars);
 }
diff --git a/src/dark_vs.ts b/src/dark_vs.ts
@@ -1,3 +1,5 @@
+// Taken from
+// https://github.com/microsoft/vscode-textmate/blob/0730e8ef740d87401764d76e9193f74c6f458b37/test-cases/themes/dark_vs.json
 export default {
   name: 'Dark Visual Studio',
   settings: [

diff --git a/src/examples.ts b/src/examples.ts
@@ -0,0 +1,52 @@
+export type SupportedLanguage = keyof typeof examples;
+
+const examples = {
+  hack: `<?hh // strict
+
+type SomeShape = shape(
+  'frame_type' => FrameType,
+  ?'commit_id' => ?string,
+);
+
+abstract final class ClassDefWithLotsOfKeywords {
+  const int BIG_NUMBER = 30000000;
+
+  public static async function genRender(): Awaitable<Stuff> {
+    return <xhp:div>
+      hello world
+    </xhp:div>;
+  }
+}
+`,
+  html: `<!DOCTYPE HTML>
+<html>
+<head>
+</head>
+<body>
+</body>
+</html>
+`,
+  javascript: `\
+const React = require('react');
+
+function create() {
+  return (
+    <div>
+      hello world
+    </div>
+  );
+}
+`,
+  python: `\
+import foo
+
+async def bar(): string:
+  f = await foo()
+  f_string = f"Hooray {f}! format strings are not supported in current Monarch grammar"
+  return foo_string
+`,
+};
+
+export function getSampleCodeForLanguage(language: keyof typeof examples): string {
+  return examples[language];
+}
diff --git a/src/index.ts b/src/index.ts
@@ -5,7 +5,7 @@ import {createOnigScanner, createOnigString, loadWASM} from 'vscode-oniguruma';
 import {INITIAL, Registry, parseRawGrammar} from 'vscode-textmate';
 type IState = monaco.languages.IState;
 
-import DEFAULT_THEME from './dark_vs';
+import DARK_VISUAL_STUDIO from './dark_vs';
 
 export async function createGrammarStore(
   scopeNameToTextMateGrammarURL: Map<string, string>,
@@ -14,7 +14,7 @@ export async function createGrammarStore(
   return new GrammarStore(registry);
 }
 
-export class GrammarStore {
+class GrammarStore {
   private scopeNameToGrammar: Map<string, Promise<IGrammar | null>> = new Map();
 
   constructor(private registry: Registry) {}
@@ -56,14 +56,9 @@ export class GrammarStore {
       },
 
       tokenizeEncoded(line: string, state: IState): monaco.languages.IEncodedLineTokens {
-        // It looks like src/vs/editor/standalone/common/monarch/monarchLexer.ts
-        // does a check to see whether state.embeddedModeData is set, and if so,
-        // performs slightly different logic?
-
         const tokenizeLineResult2 = grammar.tokenizeLine2(line, <StackElement>state);
         const endState = <IState>tokenizeLineResult2.ruleStack;
         const {tokens} = tokenizeLineResult2;
-        // convertToEndOffset(tokens, line.length);
         return {tokens, endState};
       },
     };
@@ -111,7 +106,7 @@ async function createRegistry(
 
       throw Error(`request to ${url} failed: ${response}`);
     },
-    theme: DEFAULT_THEME,
+    theme: DARK_VISUAL_STUDIO,
   });
 }
 
@@ -128,13 +123,3 @@ async function loadVSCodeOnigurumWASM(): Promise<Response | ArrayBuffer> {
   // We therefore use the non-streaming compiler :(.
   return await response.arrayBuffer();
 }
-
-// Found this function in vscode/src/vs/editor/common/model/textModelTokens.ts.
-function convertToEndOffset(tokens: Uint32Array, lineTextLength: number): void {
-  const tokenCount = tokens.length >>> 1;
-  const lastTokenIndex = tokenCount - 1;
-  for (let tokenIndex = 0; tokenIndex < lastTokenIndex; tokenIndex++) {
-    tokens[tokenIndex << 1] = tokens[(tokenIndex + 1) << 1];
-  }
-  tokens[lastTokenIndex << 1] = lineTextLength;
-}