[monaco] fix #6966: normalize textmate colors

vscode textmate tokenization indexes all kind of colors, but monaco textmate tokenization normalizes them to 6 place based hex form, so it works till a user is using normalized form in json files, if not coloring is broken

Signed-off-by: Anton Kosyakov <anton.kosyakov@typefox.io>

examples/api-tests/src/monaco-api.spec.js

@@ -25,12 +25,13 @@ describe('Monaco API', async function () {
     const { WorkspaceService } = require('@theia/workspace/lib/browser/workspace-service');
     const { MonacoEditor } = require('@theia/monaco/lib/browser/monaco-editor');
     const { MonacoResolvedKeybinding } = require('@theia/monaco/lib/browser/monaco-resolved-keybinding');
+    const { MonacoTextmateService } = require('@theia/monaco/lib/browser/textmate/monaco-textmate-service');
 
     /** @type {import('inversify').Container} */
     const container = window['theia'].container;
     const editorManager = container.get(EditorManager);
     const workspaceService = container.get(WorkspaceService);
-
+    const textmateService = container.get(MonacoTextmateService);
     /** @type {MonacoEditor} */
     let monacoEditor;
 
@@ -87,4 +88,24 @@ describe('Monaco API', async function () {
         }
     });
 
+    it('TokenizationRegistry.getColorMap', async () => {
+        if (textmateService['monacoThemeRegistry'].getThemeData().base !== 'vs') {
+            const didChangeColorMap = new Promise(resolve => {
+                const toDispose = monaco.modes.TokenizationRegistry.onDidChange(() => {
+                    toDispose.dispose();
+                    resolve();
+                })
+            });
+            textmateService['themeService'].setCurrentTheme('light');
+            await didChangeColorMap;
+        }
+
+        const textMateColorMap = textmateService['grammarRegistry'].getColorMap();
+        assert.notEqual(textMateColorMap.indexOf('#ECECEC'), -1, 'Light Theia theme should override activityBar.background color.')
+
+        const monacoColorMap = monaco.modes.TokenizationRegistry.getColorMap().
+            splice(0, textMateColorMap.length).map(c => c.toString().toUpperCase());
+        assert.deepStrictEqual(monacoColorMap, textMateColorMap, 'Text Mate colors should have the same index in the Monaco color map.');
+    });
+
 });

packages/monaco/src/browser/textmate/monaco-textmate-service.ts

@@ -74,7 +74,7 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
 
         this.grammarRegistry = new Registry({
             getOnigLib: () => this.onigasmPromise,
-            theme: this.monacoThemeRegistry.getTheme(this.currentEditorTheme),
+            theme: this.monacoThemeRegistry.getThemeData(this.currentEditorTheme),
             loadGrammar: async (scopeName: string) => {
                 const provider = this.textmateRegistry.getProvider(scopeName);
                 if (provider) {
@@ -116,7 +116,7 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
         this.toDisposeOnUpdateTheme.push(Disposable.create(() => document.body.classList.remove(currentEditorTheme)));
 
         // first update registry to run tokenization with the proper theme
-        const theme = this.monacoThemeRegistry.getTheme(currentEditorTheme);
+        const theme = this.monacoThemeRegistry.getThemeData(currentEditorTheme);
         if (theme) {
             this.grammarRegistry.setTheme(theme);
         }

packages/monaco/src/browser/textmate/monaco-theme-registry.ts

@@ -17,22 +17,37 @@
 
 // tslint:disable:no-any
 
-import { IRawTheme, Registry } from 'vscode-textmate';
+import { IRawTheme, Registry, IRawThemeSetting } from 'vscode-textmate';
 
 export interface ThemeMix extends IRawTheme, monaco.editor.IStandaloneThemeData { }
+export interface MixStandaloneTheme extends monaco.services.IStandaloneTheme {
+    themeData: ThemeMix
+}
 
 export class MonacoThemeRegistry {
 
-    protected themes = new Map<string, ThemeMix>();
+    getThemeData(): ThemeMix;
+    getThemeData(name: string): ThemeMix | undefined;
+    getThemeData(name?: string): ThemeMix | undefined {
+        const theme = this.doGetTheme(name);
+        return theme && theme.themeData;
+    }
+
+    getTheme(): MixStandaloneTheme;
+    getTheme(name: string): MixStandaloneTheme | undefined;
+    getTheme(name?: string): MixStandaloneTheme | undefined {
+        return this.doGetTheme(name);
+    }
 
-    public getTheme(name: string): IRawTheme | undefined {
-        return this.themes.get(name);
+    protected doGetTheme(name: string | undefined): MixStandaloneTheme | undefined {
+        const standaloneThemeService = monaco.services.StaticServices.standaloneThemeService.get();
+        const theme = !name ? standaloneThemeService.getTheme() : standaloneThemeService._knownThemes.get(name);
+        return theme as MixStandaloneTheme | undefined;
     }
 
     setTheme(name: string, data: ThemeMix): void {
         // monaco auto refrehes a theme with new data
         monaco.editor.defineTheme(name, data);
-        this.themes.set(name, data);
     }
 
     /**
@@ -58,8 +73,20 @@ export class MonacoThemeRegistry {
                 result.settings.push(...parentTheme.settings);
             }
         }
-        if (json.tokenColors) {
-            result.settings.push(...json.tokenColors);
+        const tokenColors: Array<IRawThemeSetting> = json.tokenColors;
+        if (Array.isArray(tokenColors)) {
+            for (const tokenColor of tokenColors) {
+                if (tokenColor.scope && tokenColor.settings) {
+                    result.settings.push({
+                        scope: tokenColor.scope,
+                        settings: {
+                            foreground: this.normalizeColor(tokenColor.settings.foreground),
+                            background: this.normalizeColor(tokenColor.settings.background),
+                            fontStyle: tokenColor.settings.fontStyle
+                        }
+                    });
+                }
+            }
         }
         if (json.colors) {
             Object.assign(result.colors, json.colors);
@@ -97,12 +124,8 @@ export class MonacoThemeRegistry {
         for (const scope of tokenColor.scope) {
 
             // Converting numbers into a format that monaco understands
-            const settings = Object.keys(tokenColor.settings).reduce((previous: { [key: string]: string }, current) => {
-                let value: string = tokenColor.settings[current];
-                if (typeof value === typeof '') {
-                    value = value.replace(/^\#/, '').slice(0, 6);
-                }
-                previous[current] = value;
+            const settings = Object.keys(tokenColor.settings).reduce((previous: { [key: string]: string | undefined }, current) => {
+                previous[current] = this.normalizeColor(tokenColor.settings[current]);
                 return previous;
             }, {});
 
@@ -111,6 +134,20 @@ export class MonacoThemeRegistry {
             });
         }
     }
+
+    protected normalizeColor(color: string | undefined): string | undefined {
+        if (!color) {
+            return undefined;
+        }
+        color = color.replace(/^\#/, '').slice(0, 6);
+        let i = 0;
+        while (color.length < 6) {
+            color = color[i] + color[i] + color.slice(i + 1);
+            i = i + 2;
+        }
+        return '#' + color;
+    }
+
 }
 
 export namespace MonacoThemeRegistry {

packages/monaco/src/typings/monaco/index.d.ts

@@ -486,10 +486,12 @@ declare module monaco.services {
     }
 
     export interface IStandaloneThemeService extends monaco.theme.IThemeService {
+        readonly _knownThemes: Map<string, IStandaloneTheme>;
         getTheme(): IStandaloneTheme;
     }
 
     export interface IStandaloneTheme {
+        themeData: monaco.editor.IStandaloneThemeData
         tokenTheme: TokenTheme;
         getColor(color: string): Color | undefined;
     }
@@ -858,6 +860,8 @@ declare module monaco.modes {
     }
     export interface TokenizationRegistry {
         get(language: string): ITokenizationSupport | null;
+        getColorMap(): monaco.color.Color[] | null;
+        readonly onDidChange: monaco.IEvent<any>;
     }
     export const TokenizationRegistry: TokenizationRegistry;