[monaco] fix #6327: fixed developer token command

Signed-off-by: Anton Kosyakov <anton.kosyakov@typefox.io>
eclipse-theia · Jan 27, 2020 · a9718db · a9718db
1 parent 8ff8bdc
commit a9718db
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 5 deletions.
diff --git a/packages/monaco/src/browser/monaco-loader.ts b/packages/monaco/src/browser/monaco-loader.ts
@@ -57,6 +57,7 @@ export function loadMonaco(vsRequire: any): Promise<void> {
                 'vs/editor/browser/editorExtensions',
                 'vs/editor/standalone/browser/simpleServices',
                 'vs/editor/standalone/browser/standaloneServices',
+                'vs/editor/standalone/browser/standaloneLanguages',
                 'vs/base/parts/quickopen/browser/quickOpenWidget',
                 'vs/base/parts/quickopen/browser/quickOpenModel',
                 'vs/base/common/filters',
@@ -77,7 +78,8 @@ export function loadMonaco(vsRequire: any): Promise<void> {
                 'vs/base/common/errors'
             ], (css: any, html: any, commands: any, actions: any,
                 keybindingsRegistry: any, keybindingResolver: any, resolvedKeybinding: any, keybindingLabels: any,
-                keyCodes: any, mime: any, editorExtensions: any, simpleServices: any, standaloneServices: any, quickOpenWidget: any, quickOpenModel: any,
+                keyCodes: any, mime: any, editorExtensions: any, simpleServices: any,
+                standaloneServices: any, standaloneLanguages: any, quickOpenWidget: any, quickOpenModel: any,
                 filters: any, styler: any, colorRegistry: any, color: any,
                 platform: any, modes: any, suggest: any, snippetParser: any,
                 configuration: any, configurationModels: any,
@@ -89,7 +91,7 @@ export function loadMonaco(vsRequire: any): Promise<void> {
                     global.monaco.commands = commands;
                     global.monaco.actions = actions;
                     global.monaco.keybindings = Object.assign({}, keybindingsRegistry, keybindingResolver, resolvedKeybinding, keybindingLabels, keyCodes);
-                    global.monaco.services = Object.assign({}, simpleServices, standaloneServices, configuration, configurationModels,
+                    global.monaco.services = Object.assign({}, simpleServices, standaloneServices, standaloneLanguages, configuration, configurationModels,
                         codeEditorService, codeEditorServiceImpl, markerService);
                     global.monaco.quickOpen = Object.assign({}, quickOpenWidget, quickOpenModel);
                     global.monaco.filters = filters;

diff --git a/packages/monaco/src/browser/textmate/monaco-textmate-service.ts b/packages/monaco/src/browser/textmate/monaco-textmate-service.ts
@@ -154,7 +154,13 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
                     throw new Error(`no grammar for ${scopeName}, ${initialLanguage}, ${JSON.stringify(configuration)}`);
                 }
                 const options = configuration.tokenizerOption ? configuration.tokenizerOption : TokenizerOption.DEFAULT;
-                monaco.languages.setTokensProvider(languageId, createTextmateTokenizer(grammar, options));
+                const tokenizer = createTextmateTokenizer(grammar, options);
+                monaco.languages.setTokensProvider(languageId, tokenizer);
+                const support = monaco.modes.TokenizationRegistry.get(languageId);
+                const themeService = monaco.services.StaticServices.standaloneThemeService.get();
+                const languageIdentifier = monaco.services.StaticServices.modeService.get().getLanguageIdentifier(languageId);
+                const adapter = new monaco.services.TokenizationSupport2Adapter(themeService, languageIdentifier!, tokenizer);
+                support!.tokenize = adapter.tokenize.bind(adapter);
             } catch (error) {
                 this.logger.warn('No grammar for this language id', languageId, error);
             }

diff --git a/packages/monaco/src/browser/textmate/textmate-tokenizer.ts b/packages/monaco/src/browser/textmate/textmate-tokenizer.ts
@@ -57,7 +57,7 @@ export namespace TokenizerOption {
     };
 }
 
-export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOption): monaco.languages.EncodedTokensProvider {
+export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOption): monaco.languages.EncodedTokensProvider & monaco.languages.TokensProvider {
     if (options.lineLimit !== undefined && (options.lineLimit <= 0 || !Number.isInteger(options.lineLimit))) {
         throw new Error(`The 'lineLimit' must be a positive integer. It was ${options.lineLimit}.`);
     }
@@ -74,6 +74,21 @@ export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOpt
                 endState: new TokenizerState(result.ruleStack),
                 tokens: result.tokens
             };
+        },
+        tokenize(line: string, state: TokenizerState): monaco.languages.ILineTokens {
+            let processedLine = line;
+            if (options.lineLimit !== undefined && line.length > options.lineLimit) {
+                // Line is too long to be tokenized
+                processedLine = line.substr(0, options.lineLimit);
+            }
+            const result = grammar.tokenizeLine(processedLine, state.ruleStack);
+            return {
+                endState: new TokenizerState(result.ruleStack),
+                tokens: result.tokens.map(t => ({
+                    startIndex: t.startIndex,
+                    scopes: t.scopes.reverse().join(' ')
+                }))
+            };
         }
     };
 }
diff --git a/packages/monaco/src/typings/monaco/index.d.ts b/packages/monaco/src/typings/monaco/index.d.ts
@@ -438,6 +438,11 @@ declare module monaco.keybindings {
 
 declare module monaco.services {
 
+    export class TokenizationSupport2Adapter implements monaco.modes.ITokenizationSupport {
+        constructor(standaloneThemeService: IStandaloneThemeService, languageIdentifier: LanguageIdentifier, actual: monaco.languages.TokensProvider)
+        tokenize(line: string, state: monaco.languages.IState, offsetDelta: number): any;
+    }
+
     export const ICodeEditorService: any;
     export const IConfigurationService: any;
 
@@ -511,7 +516,8 @@ declare module monaco.services {
 
     // https://github.com/TypeFox/vscode/blob/70b8db24a37fafc77247de7f7cb5bb0195120ed0/src/vs/editor/common/services/modeService.ts#L30
     export interface IModeService {
-        createByFilepathOrFirstLine(rsource: monaco.Uri | null, firstLine?: string): ILanguageSelection
+        createByFilepathOrFirstLine(rsource: monaco.Uri | null, firstLine?: string): ILanguageSelection;
+        getLanguageIdentifier(modeId: string): LanguageIdentifier | null;
     }
 
     export interface ILanguageSelection {
@@ -847,6 +853,14 @@ declare module monaco.editorExtensions {
 }
 declare module monaco.modes {
 
+    export interface ITokenizationSupport {
+        tokenize(line: string, state: monaco.languages.IState, offsetDelta: number): any;
+    }
+    export interface TokenizationRegistry {
+        get(language: string): ITokenizationSupport | null;
+    }
+    export const TokenizationRegistry: TokenizationRegistry;
+
     export class TokenMetadata {
 
         public static getLanguageId(metadata: number): number;