Improve plugin node.js error handling

Signed-off-by: Yevhen Vydolob <yvydolob@redhat.com>
eclipse-theia · Jul 12, 2019 · 5b48e66 · 5b48e66
1 parent 2fa4738
commit 5b48e66
Show file tree

Hide file tree

Showing 2 changed files with 71 additions and 3 deletions.
diff --git a/packages/plugin-ext/src/hosted/node/hosted-plugin-process.ts b/packages/plugin-ext/src/hosted/node/hosted-plugin-process.ts
@@ -17,7 +17,7 @@
 import * as path from 'path';
 import * as cp from 'child_process';
 import { injectable, inject, named } from 'inversify';
-import { ILogger, ConnectionErrorHandler, ContributionProvider } from '@theia/core/lib/common';
+import { ILogger, ConnectionErrorHandler, ContributionProvider, MessageService } from '@theia/core/lib/common';
 import { Emitter } from '@theia/core/lib/common/event';
 import { createIpcEnv } from '@theia/core/lib/node/messaging/ipc-protocol';
 import { HostedPluginClient, ServerPluginRunner, PluginMetadata, PluginHostEnvironmentVariable } from '../../common/plugin-protocol';
@@ -49,10 +49,15 @@ export class HostedPluginProcess implements ServerPluginRunner {
     @named(PluginHostEnvironmentVariable)
     protected readonly pluginHostEnvironmentVariables: ContributionProvider<PluginHostEnvironmentVariable>;
 
+    @inject(MessageService)
+    protected readonly messageService: MessageService;
+
     private childProcess: cp.ChildProcess | undefined;
 
     private client: HostedPluginClient;
 
+    private terminatingPluginServer = false;
+
     private async getClientId(): Promise<number> {
         return await this.pluginProcessCache.getLazyClientId(this.client);
     }
@@ -103,6 +108,8 @@ export class HostedPluginProcess implements ServerPluginRunner {
         if (this.childProcess === undefined) {
             return;
         }
+
+        this.terminatingPluginServer = true;
         // tslint:disable-next-line:no-shadowed-variable
         const cp = this.childProcess;
         this.childProcess = undefined;
@@ -131,6 +138,7 @@ export class HostedPluginProcess implements ServerPluginRunner {
         if (this.childProcess) {
             this.terminatePluginServer();
         }
+        this.terminatingPluginServer = false;
         this.childProcess = this.fork({
             serverName: 'hosted-plugin',
             logger: this.logger,
@@ -184,11 +192,23 @@ export class HostedPluginProcess implements ServerPluginRunner {
         childProcess.stderr.on('data', data => this.logger.error(`[${options.serverName}: ${childProcess.pid}] ${data.toString().trim()}`));
 
         this.logger.debug(`[${options.serverName}: ${childProcess.pid}] IPC started`);
-        childProcess.once('exit', () => this.logger.debug(`[${options.serverName}: ${childProcess.pid}] IPC exited`));
-
+        childProcess.once('exit', (code: number, signal: string) => this.onChildProcessExit(options.serverName, childProcess.pid, code, signal));
+        childProcess.on('error', err => this.onChildProcessError(err));
         return childProcess;
     }
 
+    private onChildProcessExit(serverName: string, pid: number, code: number, signal: string): void {
+        if (this.terminatingPluginServer) {
+            return;
+        }
+        this.logger.error(`[${serverName}: ${pid}] IPC exited, with signal: ${signal}, and exit code: ${code}`);
+        this.messageService.error('Plugin runtime crashed unexpectedly, all plugins are not working, please reload...', { timeout: 15 * 60 * 1000 });
+    }
+
+    private onChildProcessError(err: Error): void {
+        this.logger.error(`Error from plugin host: ${err.message}`);
+    }
+
     async getExtraPluginMetadata(): Promise<PluginMetadata[]> {
         return [];
     }

diff --git a/packages/plugin-ext/src/hosted/node/plugin-host.ts b/packages/plugin-ext/src/hosted/node/plugin-host.ts
@@ -19,6 +19,54 @@ import { RPCProtocolImpl } from '../../api/rpc-protocol';
 import { PluginHostRPC } from './plugin-host-rpc';
 console.log('PLUGIN_HOST(' + process.pid + ') starting instance');
 
+// override exit() function, to do not allow plugin kill this node
+process.exit = function (code?: number) {
+    const err = new Error('An plugin call process.exit() and it was prevented.');
+    console.warn(err.stack);
+} as (code?: number) => never;
+
+// same for 'crash'(works only in electron)
+// tslint:disable-next-line: no-any
+const proc = process as any;
+if (proc.crash) {
+    proc.crash = function () {
+        const err = new Error('An plugin call process.crash() and it was prevented.');
+        console.warn(err.stack);
+    };
+}
+
+process.on('uncaughtException', (err: Error) => {
+    console.error(err);
+});
+
+// tslint:disable-next-line: no-any
+const unhandledPromises: Promise<any>[] = [];
+
+// tslint:disable-next-line: no-any
+process.on('unhandledRejection', (reason: any, promise: Promise<any>) => {
+    unhandledPromises.push(promise);
+    setTimeout(() => {
+        const index = unhandledPromises.indexOf(promise);
+        if (index >= 0) {
+            promise.catch(err => {
+                unhandledPromises.splice(index, 1);
+                console.error(`Promise rejection not handled in one second: ${err}`);
+                if (err.stack) {
+                    console.error(`With stack trace: ${err.stack}`);
+                }
+            });
+        }
+    }, 1000);
+});
+
+// tslint:disable-next-line: no-any
+process.on('rejectionHandled', (promise: Promise<any>) => {
+    const index = unhandledPromises.indexOf(promise);
+    if (index >= 0) {
+        unhandledPromises.splice(index, 1);
+    }
+});
+
 const emitter = new Emitter();
 const rpc = new RPCProtocolImpl({
     onMessage: emitter.event,