Merge pull request #15 from jupyterlite/wasm-with-webworker

Use WASM commands running in webworker

package.json

@@ -30,12 +30,13 @@
         "url": "https://github.com/jupyterlite/terminal.git"
     },
     "scripts": {
-        "build": "jlpm build:lib && jlpm build:labextension:dev",
-        "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:labextension",
+        "build": "jlpm build:lib && jlpm build:worker && jlpm build:labextension:dev",
+        "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:worker && jlpm build:labextension",
         "build:labextension": "jupyter labextension build .",
         "build:labextension:dev": "jupyter labextension build --development True .",
         "build:lib": "tsc --sourceMap",
         "build:lib:prod": "tsc",
+        "build:worker": "webpack --config worker.webpack.config.js --mode=development",
         "clean": "jlpm clean:lib",
         "clean:lib": "rimraf lib tsconfig.tsbuildinfo",
         "clean:lintcache": "rimraf .eslintcache .stylelintcache",
@@ -57,12 +58,15 @@
         "watch:labextension": "jupyter labextension watch ."
     },
     "dependencies": {
+        "@jupyterlab/coreutils": "^6.2.2",
         "@jupyterlab/services": "^7.2.0",
         "@jupyterlab/terminal": "^4.2.0",
         "@jupyterlab/terminal-extension": "^4.2.0",
-        "@jupyterlite/cockle": "^0.0.3",
+        "@jupyterlite/cockle": "^0.0.4",
+        "@jupyterlite/contents": "^0.3.0 || ^0.4.0-beta.0",
         "@jupyterlite/server": "^0.3.0 || ^0.4.0-beta.0",
         "@lumino/coreutils": "^2.1.2",
+        "comlink": "^4.4.1",
         "mock-socket": "^9.3.1"
     },
     "devDependencies": {
@@ -90,6 +94,8 @@
         "stylelint-csstree-validator": "^3.0.0",
         "stylelint-prettier": "^4.0.0",
         "typescript": "~5.0.2",
+        "webpack": "^5.87.0",
+        "webpack-cli": "^5.1.4",
         "yjs": "^13.5.0"
     },
     "sideEffects": [

src/index.ts

@@ -24,7 +24,7 @@ const terminalsPlugin: JupyterLiteServerPlugin<ITerminals> = {
     );
 
     const { serviceManager } = app;
-    const { contents, serverSettings, terminals } = serviceManager;
+    const { serverSettings, terminals } = serviceManager;
     console.log('terminals available:', terminals.isAvailable());
     console.log('terminals ready:', terminals.isReady); // Not ready
     console.log('terminals active:', terminals.isActive);
@@ -33,7 +33,7 @@ const terminalsPlugin: JupyterLiteServerPlugin<ITerminals> = {
     await terminals.ready;
     console.log('terminals ready after await:', terminals.isReady); // Ready
 
-    return new Terminals(serverSettings.wsUrl, contents);
+    return new Terminals(serverSettings.wsUrl);
   }
 };
 

src/terminal.ts

@@ -1,64 +1,78 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
-import { JupyterFileSystem, Shell, IFileSystem } from '@jupyterlite/cockle';
-
 import { JSONPrimitive } from '@lumino/coreutils';
 
 import {
   Server as WebSocketServer,
   Client as WebSocketClient
 } from 'mock-socket';
 
-import { ITerminal } from './tokens';
+import { wrap } from 'comlink';
+
+import { ITerminal, IRemoteWorkerTerminal } from './tokens';
 
 export class Terminal implements ITerminal {
   /**
    * Construct a new Terminal.
    */
-  constructor(options: ITerminal.IOptions) {
-    this._name = options.name;
-    this._fs = new JupyterFileSystem(options.contentsManager);
-    console.log('==> new Terminal', this._name, this._fs);
+  constructor(readonly options: ITerminal.IOptions) {
+    this._initWorker();
+  }
+
+  private async _initWorker(): Promise<void> {
+    this._worker = new Worker(new URL('./worker.js', import.meta.url), {
+      type: 'module'
+    });
+
+    this._remote = wrap(this._worker);
+    const { baseUrl } = this.options;
+    await this._remote.initialize({ baseUrl });
+  }
+
+  /**
+   * Process a message coming from the JavaScript web worker.
+   *
+   * @param msg The worker message to process.
+   */
+  private _processWorkerMessage(msg: any, socket: WebSocketClient): void {
+    if (msg.type === 'output') {
+      const ret = JSON.stringify(['stdout', msg.text]);
+      socket.send(ret);
+    }
   }
 
   /**
    * Get the name of the terminal.
    */
   get name(): string {
-    return this._name;
+    return this.options.name;
   }
 
   async wsConnect(url: string) {
     console.log('==> Terminal.wsConnect', url);
 
-    // const server = new WebSocketServer(url, { mock: false });
     const server = new WebSocketServer(url);
 
     server.on('connection', async (socket: WebSocketClient) => {
       console.log('==> server connection', this, socket);
 
-      const outputCallback = async (output: string) => {
-        console.log('==> recv from shell:', output);
-        const ret = JSON.stringify(['stdout', output]);
-        socket.send(ret);
+      this._worker!.onmessage = e => {
+        this._processWorkerMessage(e.data, socket);
       };
 
-      this._shell = new Shell(this._fs, outputCallback);
-      console.log('==> shell', this._shell);
-
       socket.on('message', async (message: any) => {
         const data = JSON.parse(message) as JSONPrimitive[];
-        console.log('==> socket message', data);
+        //console.log('==> socket message', data);
         const message_type = data[0];
         const content = data.slice(1);
 
         if (message_type === 'stdin') {
-          await this._shell!.input(content[0] as string);
+          await this._remote!.input(content[0] as string);
         } else if (message_type === 'set_size') {
           const rows = content[0] as number;
           const columns = content[1] as number;
-          await this._shell!.setSize(rows, columns);
+          await this._remote!.setSize(rows, columns);
         }
       });
 
@@ -75,11 +89,10 @@ export class Terminal implements ITerminal {
       console.log('==> Returning handshake via socket', res);
       socket.send(res);
 
-      await this._shell!.start();
+      await this._remote!.start();
     });
   }
 
-  private _name: string;
-  private _fs: IFileSystem;
-  private _shell?: Shell;
+  private _worker?: Worker;
+  private _remote?: IRemoteWorkerTerminal;
 }

src/terminals.ts

@@ -1,7 +1,8 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
-import { Contents, TerminalAPI } from '@jupyterlab/services';
+import { PageConfig } from '@jupyterlab/coreutils';
+import { TerminalAPI } from '@jupyterlab/services';
 
 import { Terminal } from './terminal';
 import { ITerminals } from './tokens';
@@ -13,14 +14,9 @@ export class Terminals implements ITerminals {
   /**
    * Construct a new Terminals object.
    */
-  constructor(wsUrl: string, contentsManager: Contents.IManager) {
+  constructor(wsUrl: string) {
     this._wsUrl = wsUrl;
-    this._contentsManager = contentsManager;
-    console.log(
-      '==> Terminals.constructor',
-      this._wsUrl,
-      this._contentsManager
-    );
+    console.log('==> Terminals.constructor', this._wsUrl);
   }
 
   /**
@@ -40,7 +36,8 @@ export class Terminals implements ITerminals {
   async startNew(): Promise<TerminalAPI.IModel> {
     const name = this._nextAvailableName();
     console.log('==> Terminals.new', name);
-    const term = new Terminal({ name, contentsManager: this._contentsManager });
+    const baseUrl = PageConfig.getBaseUrl();
+    const term = new Terminal({ name, baseUrl });
     this._terminals.set(name, term);
 
     const url = `${this._wsUrl}terminals/websocket/${name}`;
@@ -59,6 +56,5 @@ export class Terminals implements ITerminals {
   }
 
   private _wsUrl: string;
-  private _contentsManager: Contents.IManager;
   private _terminals: Map<string, Terminal> = new Map();
 }

src/tokens.ts

@@ -1,10 +1,12 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
 
-import { Contents, TerminalAPI } from '@jupyterlab/services';
+import { TerminalAPI } from '@jupyterlab/services';
 
 import { Token } from '@lumino/coreutils';
 
+import { Remote } from 'comlink';
+
 /**
  * The token for the Terminals service.
  */
@@ -50,6 +52,33 @@ export namespace ITerminal {
      */
     name: string;
 
-    contentsManager: Contents.IManager;
+    baseUrl: string;
+  }
+}
+
+export interface IWorkerTerminal {
+  input(text: string): Promise<void>;
+  setSize(rows: number, columns: number): Promise<void>;
+  start(): Promise<void>;
+}
+
+export namespace IWorkerTerminal {
+  /**
+   * Initialization options for a worker.
+   */
+  export interface IOptions {
+    baseUrl: string;
   }
 }
+
+export interface IRemote extends IWorkerTerminal {
+  /**
+   * Handle any lazy initialization activities.
+   */
+  initialize(options: IWorkerTerminal.IOptions): Promise<void>;
+}
+
+/**
+ * An convenience interface for Pyodide workers wrapped by a comlink Remote.
+ */
+export type IRemoteWorkerTerminal = Remote<IRemote>;

src/worker.ts

@@ -0,0 +1,63 @@
+import { Shell } from '@jupyterlite/cockle';
+import { DriveFS } from '@jupyterlite/contents';
+
+import { expose } from 'comlink';
+
+import { IWorkerTerminal } from './tokens';
+
+class WorkerTerminal implements IWorkerTerminal {
+  async initialize(options: IWorkerTerminal.IOptions): Promise<void> {
+    this._options = options;
+    console.log('WorkerTerminal.initialize', this._options, this._wantDriveFS);
+  }
+
+  async input(text: string): Promise<void> {
+    await this._shell!.input(text);
+  }
+
+  async setSize(rows: number, columns: number): Promise<void> {
+    await this._shell!.setSize(rows, columns);
+  }
+
+  async start(): Promise<void> {
+    this._shell = new Shell(this.output, this._mountpoint);
+    const { FS, PATH, ERRNO_CODES } = await this._shell.initFilesystem();
+
+    if (this._wantDriveFS) {
+      this._driveFS = new DriveFS({
+        FS,
+        PATH,
+        ERRNO_CODES,
+        baseUrl: this._options!.baseUrl,
+        driveName: '',
+        mountpoint: this._mountpoint
+      });
+      FS.mount(this._driveFS, {}, this._mountpoint);
+      FS.chdir(this._mountpoint);
+    } else {
+      // Add some dummy files if not using DriveFS.
+      FS.writeFile('file.txt', 'This is the contents of the file');
+      FS.writeFile('other.txt', 'Some other file');
+      FS.mkdir('dir');
+    }
+
+    await this._shell.start();
+  }
+
+  private async output(text: string): Promise<void> {
+    postMessage({
+      type: 'output',
+      text: text
+    });
+  }
+
+  private _options: IWorkerTerminal.IOptions | null = null;
+  private _shell?: Shell;
+  private _mountpoint: string = '/drive';
+  private _wantDriveFS: boolean = true;
+  private _driveFS?: DriveFS;
+}
+
+const obj = new WorkerTerminal();
+
+expose(obj);

tsconfig.json

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+    "allowJs": true,
     "allowSyntheticDefaultImports": true,
     "composite": true,
     "declaration": true,

worker.webpack.config.js

@@ -0,0 +1,33 @@
+const path = require('path');
+const rules = [
+  {
+    test: /\.js$/,
+    exclude: /node_modules/,
+    loader: 'source-map-loader'
+  }
+];
+
+const resolve = {
+  fallback: {
+    fs: false,
+    child_process: false,
+    crypto: false
+  },
+  extensions: ['.js']
+};
+
+module.exports = [
+  {
+    entry: './lib/worker.js',
+    output: {
+      filename: 'worker.js',
+      path: path.resolve(__dirname, 'lib'),
+      libraryTarget: 'amd'
+    },
+    module: {
+      rules
+    },
+    devtool: 'source-map',
+    resolve
+  }
+];