don't block web socket with many plugins

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Change Log
 
+## v0.12.0
+
+Breaking changes:
+
+- [plugin] don't block web socket with many plugins [6252](https://github.com/eclipse-theia/theia/pull/6252)
+  - `PluginModel` does not have anymore `contributes` and `dependencies` to avoid sending unnecessary data.
+    - Use `PluginReader.readContribution` to load contributes.
+    - Use `PluginReader.readDependencies` to load dependencies.
+  - `PluginMetadata` does not have anymore raw package.json model to avoid sending excessive data to the frontend.
+    - `theia.Plugin.packageJSON` throws an unsupported error for frontend plugins as a consequence. Please convert to a backend plugin if you need access to it.
+  - `PluginManagerExt.$init` does not start plugins anymore, but only initialize the manager RPC services to avoid sending excessive initialization data, as all preferences, on each deployment.
+    - Please call `$start` to start plugins.
+  - `PluginDeployerHandler.getPluginMetadata` is replaced with `PluginDeployerHandler.getPluginDependencies` to access plugin dependencies.
+  - `HostedPluginServer.getDeployedMetadata` is replaced with `HostedPluginServer.getDeployedPluginIds` and `HostedPluginServer.getDeployedPlugins`
+  to fetch first only ids of deployed plugins and then deployed metadata for only yet not loaded plugins.
+  - `HostedPluginDeployerHandler.getDeployedFrontendMetadata` and `HostedPluginDeployerHandler.getDeployedBackendMetadata` are replaced with 
+  `HostedPluginDeployerHandler.getDeployedFrontendPluginIds`, `HostedPluginDeployerHandlergetDeployedBackendPluginIds` and `HostedPluginDeployerHandler.getDeployedPlugin` to featch first only ids and then deplyoed metadata fro only yet not loaded plugins.
+  - `PluginHost.init` can initialize plugins asynchronous, synchronous initialization is still supported.
+  - `HostedPluginReader.doGetPluginMetadata` is renamed to `HostedPluginReader.getPluginMetadata`.
+  - `PluginDebugAdapterContribution.languages`, `PluginDebugAdapterContribution.getSchemaAttributes` and `PluginDebugAdapterContribution.getConfigurationSnippets` are removed to prevent sending the contributions second time to the frontend. Debug contributions are loaded statically from the deployed plugin metadata instead. The same for corresponding methods in `DebugExtImpl`.
+
 ## v0.11.0
 
 - [core] added <kbd>ENTER</kbd> event handler to the open button in explorer [#6158](https://github.com/eclipse-theia/theia/pull/6158)

packages/core/package.json

@@ -43,7 +43,7 @@
     "vscode-languageserver-types": "^3.15.0-next",
     "vscode-uri": "^1.0.8",
     "vscode-ws-jsonrpc": "^0.1.1",
-    "ws": "^5.2.2",
+    "ws": "^7.1.2",
     "yargs": "^11.1.0"
   },
   "publishConfig": {

packages/core/src/node/messaging/messaging-contribution.ts

@@ -84,7 +84,10 @@ export class MessagingContribution implements BackendApplicationContribution, Me
     onStart(server: http.Server | https.Server): void {
         const wss = new ws.Server({
             server,
-            perMessageDeflate: false
+            perMessageDeflate: {
+                // don't compress if a message is less than 256kb
+                threshold: 256 * 1024
+            }
         });
         interface CheckAliveWS extends ws {
             alive: boolean;

packages/monaco/src/browser/textmate/monaco-textmate-service.ts

@@ -23,8 +23,6 @@ import { LanguageGrammarDefinitionContribution, getEncodedLanguageId } from './t
 import { createTextmateTokenizer, TokenizerOption } from './textmate-tokenizer';
 import { TextmateRegistry } from './textmate-registry';
 import { MonacoThemeRegistry } from './monaco-theme-registry';
-import { MonacoEditor } from '../monaco-editor';
-import { EditorManager } from '@theia/editor/lib/browser';
 
 export const OnigasmPromise = Symbol('OnigasmPromise');
 export type OnigasmPromise = Promise<IOnigLib>;
@@ -60,9 +58,6 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
     @inject(MonacoThemeRegistry)
     protected readonly monacoThemeRegistry: MonacoThemeRegistry;
 
-    @inject(EditorManager)
-    private readonly editorManager: EditorManager;
-
     initialize(): void {
         if (!isBasicWasmSupported) {
             console.log('Textmate support deactivated because WebAssembly is not detected.');
@@ -109,7 +104,6 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
         for (const { id } of monaco.languages.getLanguages()) {
             monaco.languages.onLanguage(id, () => this.activateLanguage(id));
         }
-        this.detectLanguages();
     }
 
     protected readonly toDisposeOnUpdateTheme = new DisposableCollection();
@@ -166,11 +160,4 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
         }
     }
 
-    detectLanguages(): void {
-        for (const editor of MonacoEditor.getAll(this.editorManager)) {
-            if (editor.languageAutoDetected) {
-                editor.detectLanguage();
-            }
-        }
-    }
 }

packages/plugin-dev/src/node/hosted-plugin-reader.ts

@@ -34,7 +34,7 @@ export class HostedPluginReader implements BackendApplicationContribution {
     protected deployerHandler: HostedPluginDeployerHandler;
 
     async initialize(): Promise<void> {
-        this.pluginReader.doGetPluginMetadata(process.env.HOSTED_PLUGIN)
+        this.pluginReader.getPluginMetadata(process.env.HOSTED_PLUGIN)
             .then(this.hostedPlugin.resolve.bind(this.hostedPlugin));
 
         const pluginPath = process.env.HOSTED_PLUGIN;

packages/plugin-ext-vscode/src/node/plugin-vscode-init.ts

@@ -44,7 +44,8 @@ export const doInitialization: BackendInitializationFn = (apiFactory: PluginAPIF
     vscode.commands.registerCommand = function (command: theia.CommandDescription | string, handler?: <T>(...args: any[]) => T | Thenable<T>, thisArg?: any): any {
         // use of the ID when registering commands
         if (typeof command === 'string') {
-            const commands = plugin.model.contributes && plugin.model.contributes.commands;
+            const rawCommands = plugin.rawModel.contributes && plugin.rawModel.contributes.commands;
+            const commands = rawCommands ? Array.isArray(rawCommands) ? rawCommands : [rawCommands] : undefined;
             if (handler && commands && commands.some(item => item.command === command)) {
                 return vscode.commands.registerHandler(command, handler, thisArg);
             }

packages/plugin-ext-vscode/src/node/scanner-vscode.ts

@@ -34,6 +34,7 @@ export class VsCodePluginScanner extends TheiaPluginScanner implements PluginSca
             plugin.name = plugin.name.substr(built_prefix.length);
         }
         const result: PluginModel = {
+            packagePath: plugin.packagePath,
             // see id definition: https://github.com/microsoft/vscode/blob/15916055fe0cb9411a5f36119b3b012458fe0a1d/src/vs/platform/extensions/common/extensions.ts#L167-L169
             id: `${plugin.publisher.toLowerCase()}.${plugin.name.toLowerCase()}`,
             name: plugin.name,
@@ -47,13 +48,26 @@ export class VsCodePluginScanner extends TheiaPluginScanner implements PluginSca
             },
             entryPoint: {
                 backend: plugin.main
-            },
-            extensionDependencies: this.getDeployableDependencies(plugin.extensionDependencies || [])
+            }
         };
-        result.contributes = this.readContributions(plugin);
         return result;
     }
 
+    /**
+     * Maps extension dependencies to deployable extension dependencies.
+     */
+    getDependencies(plugin: PluginPackage): Map<string, string> | undefined {
+        if (!plugin.extensionDependencies || !plugin.extensionDependencies.length) {
+            return undefined;
+        }
+        const dependencies = new Map<string, string>();
+        for (const dependency of plugin.extensionDependencies) {
+            const dependencyId = dependency.toLowerCase();
+            dependencies.set(dependencyId, this.VSCODE_PREFIX + dependencyId);
+        }
+        return dependencies;
+    }
+
     getLifecycle(plugin: PluginPackage): PluginLifecycle {
         return {
             startMethod: 'activate',
@@ -63,11 +77,4 @@ export class VsCodePluginScanner extends TheiaPluginScanner implements PluginSca
         };
     }
 
-    /**
-     * Converts an array of extension dependencies
-     * to an array of deployable extension dependencies
-     */
-    private getDeployableDependencies(dependencies: string[]): string[] {
-        return dependencies.map(dep => this.VSCODE_PREFIX + dep.toLowerCase());
-    }
 }

packages/plugin-ext/src/common/plugin-api-rpc.ts

@@ -66,7 +66,6 @@ import {
 import { ExtPluginApi } from './plugin-ext-api-contribution';
 import { KeysToAnyValues, KeysToKeysToAnyValue } from './types';
 import { CancellationToken, Progress, ProgressOptions } from '@theia/plugin';
-import { IJSONSchema, IJSONSchemaSnippet } from '@theia/core/lib/common/json-schema';
 import { DebuggerDescription } from '@theia/debug/lib/common/debug-service';
 import { DebugProtocol } from 'vscode-debugprotocol';
 import { SymbolInformation } from 'vscode-languageserver-types';
@@ -75,16 +74,6 @@ import { MaybePromise } from '@theia/core/lib/common/types';
 import { QuickOpenItem, QuickOpenItemOptions } from '@theia/core/lib/common/quick-open-model';
 import { QuickTitleButton } from '@theia/core/lib/common/quick-open-model';
 
-export interface PluginInitData {
-    plugins: PluginMetadata[];
-    preferences: PreferenceData;
-    globalState: KeysToKeysToAnyValue;
-    workspaceState: KeysToKeysToAnyValue;
-    env: EnvInit;
-    extApi?: ExtPluginApi[];
-    activationEvents: string[]
-}
-
 export interface PreferenceData {
     [scope: number]: any;
 }
@@ -99,7 +88,7 @@ export interface Plugin {
 
 export interface ConfigStorage {
     hostLogPath: string;
-    hostStoragePath: string,
+    hostStoragePath?: string,
 }
 
 export interface EnvInit {
@@ -141,6 +130,7 @@ export const emptyPlugin: Plugin = {
             type: 'empty',
             version: 'empty'
         },
+        packagePath: 'empty',
         entryPoint: {
 
         }
@@ -161,12 +151,35 @@ export const emptyPlugin: Plugin = {
     }
 };
 
+export interface PluginManagerInitializeParams {
+    preferences: PreferenceData
+    globalState: KeysToKeysToAnyValue
+    workspaceState: KeysToKeysToAnyValue
+    env: EnvInit
+    extApi?: ExtPluginApi[]
+}
+
+export interface PluginManagerStartParams {
+    plugins: PluginMetadata[]
+    configStorage: ConfigStorage
+    activationEvents: string[]
+}
+
 export interface PluginManagerExt {
-    $stop(pluginId?: string): PromiseLike<void>;
 
-    $init(pluginInit: PluginInitData, configStorage: ConfigStorage): PromiseLike<void>;
+    /** initialize the manager, should be called only once */
+    $init(params: PluginManagerInitializeParams): Promise<void>;
+
+    /** load and activate plugins */
+    $start(params: PluginManagerStartParams): Promise<void>;
+
+    /** deactivate the plugin */
+    $stop(pluginId: string): Promise<void>;
+
+    /** deactivate all plugins */
+    $stop(): Promise<void>;
 
-    $updateStoragePath(path: string | undefined): PromiseLike<void>;
+    $updateStoragePath(path: string | undefined): Promise<void>;
 
     $activateByEvent(event: string): Promise<void>;
 }
@@ -1241,9 +1254,6 @@ export interface DebugExt {
     $sessionDidChange(sessionId: string | undefined): void;
     $provideDebugConfigurations(debugType: string, workspaceFolder: string | undefined): Promise<theia.DebugConfiguration[]>;
     $resolveDebugConfigurations(debugConfiguration: theia.DebugConfiguration, workspaceFolder: string | undefined): Promise<theia.DebugConfiguration | undefined>;
-    $getSupportedLanguages(debugType: string): Promise<string[]>;
-    $getSchemaAttributes(debugType: string): Promise<IJSONSchema[]>;
-    $getConfigurationSnippets(debugType: string): Promise<IJSONSchemaSnippet[]>;
     $createDebugSession(debugConfiguration: theia.DebugConfiguration): Promise<string>;
     $terminateDebugSession(sessionId: string): Promise<void>;
     $getTerminalCreationOptions(debugType: string): Promise<TerminalOptionsExt | undefined>;

packages/plugin-ext/src/common/plugin-protocol.ts

@@ -228,6 +228,14 @@ export interface PluginScanner {
      * @returns {PluginLifecycle}
      */
     getLifecycle(plugin: PluginPackage): PluginLifecycle;
+
+    getContribution(plugin: PluginPackage): PluginContribution | undefined;
+
+    /**
+     * A mapping between a dependency as its defined in package.json
+     * and its deployable form, e.g. `publisher.name` -> `vscode:extension/publisher.name`
+     */
+    getDependencies(plugin: PluginPackage): Map<string, string> | undefined;
 }
 
 export const PluginDeployer = Symbol('PluginDeployer');
@@ -374,22 +382,20 @@ export interface PluginModel {
         type: PluginEngine;
         version: string;
     };
-    entryPoint: {
-        frontend?: string;
-        backend?: string;
-    };
-    contributes?: PluginContribution;
-    /**
-     * The deployable form of extensionDependencies from package.json,
-     * i.e. not `publisher.name`, but `vscode:extension/publisher.name`.
-     */
-    extensionDependencies?: string[];
+    entryPoint: PluginEntryPoint;
+    packagePath: string;
+}
+
+export interface PluginEntryPoint {
+    frontend?: string;
+    backend?: string;
 }
 
 /**
  * This interface describes some static plugin contributions.
  */
 export interface PluginContribution {
+    activationEvents?: string[];
     configuration?: PreferenceSchema[];
     configurationDefaults?: PreferenceSchemaProperties;
     languages?: LanguageContribution[];
@@ -583,7 +589,6 @@ export interface ExtensionContext {
 
 export interface PluginMetadata {
     host: string;
-    source: PluginPackage;
     model: PluginModel;
     lifecycle: PluginLifecycle;
 }
@@ -610,20 +615,34 @@ export interface HostedPluginClient {
     onDidDeploy(): void;
 }
 
+export interface PluginDependencies {
+    metadata: PluginMetadata
+    mapping?: Map<string, string>
+}
+
 export const PluginDeployerHandler = Symbol('PluginDeployerHandler');
 export interface PluginDeployerHandler {
     deployFrontendPlugins(frontendPlugins: PluginDeployerEntry[]): Promise<void>;
     deployBackendPlugins(backendPlugins: PluginDeployerEntry[]): Promise<void>;
 
-    getPluginMetadata(pluginToBeInstalled: PluginDeployerEntry): Promise<PluginMetadata | undefined>
+    getPluginDependencies(pluginToBeInstalled: PluginDeployerEntry): Promise<PluginDependencies | undefined>
+}
+
+export interface GetDeployedPluginsParams {
+    pluginIds: string[]
+}
+
+export interface DeployedPlugin {
+    metadata: PluginMetadata;
+    contributes?: PluginContribution;
 }
 
 export const HostedPluginServer = Symbol('HostedPluginServer');
 export interface HostedPluginServer extends JsonRpcServer<HostedPluginClient> {
 
-    getDeployedMetadata(): Promise<PluginMetadata[]>;
-    getDeployedFrontendMetadata(): Promise<PluginMetadata[]>;
-    getDeployedBackendMetadata(): Promise<PluginMetadata[]>;
+    getDeployedPluginIds(): Promise<string[]>;
+
+    getDeployedPlugins(params: GetDeployedPluginsParams): Promise<DeployedPlugin[]>;
 
     getExtPluginAPI(): Promise<ExtPluginApi[]>;