Support for debuggable web tests (#10326)

.vscode/launch.json

@@ -99,6 +99,27 @@
         },
         {
             "name": "Web Tests",
+            "type": "extensionHost",
+            "debugWebWorkerHost": true,
+            "request": "launch",
+            "args": [
+                "${workspaceFolder}/src/test/datascience",
+                "--extensionDevelopmentPath=${workspaceFolder}",
+                "--enable-proposed-api",
+                "--extensionDevelopmentKind=web",
+                "--extensionTestsPath=${workspaceFolder}/out/extension.web.bundle"
+            ],
+            "outFiles": ["${workspaceFolder}/out/**/*.*"],
+            "sourceMaps": true,
+            "preLaunchTask": "Start Jupyter Server",
+            "postDebugTask": "Stop Jupyter Server",
+            "presentation": {
+                "group": "2_tests",
+                "order": 11
+            }
+        },
+        {
+            "name": "Web Tests (without debugging)",
             "type": "node",
             "program": "${workspaceFolder}/build/launchWebTest.js",
             "request": "launch",
@@ -108,6 +129,9 @@
             "presentation": {
                 "group": "2_tests",
                 "order": 11
+            },
+            "env": {
+                "CI_PYTHON_PATH": "" // Update with path to real python interpereter used for testing.
             }
         },
         {

.vscode/tasks.json

@@ -77,6 +77,39 @@
             "group": {
                 "kind": "build"
             }
+        },
+        {
+            "label": "Start Jupyter Server",
+            "type": "npm",
+            "dependsOn": "compile-web-test",
+            "isBackground": false,
+            "script": "startJupyterServer",
+            "problemMatcher": [],
+            "options": {
+                "env": {
+                    "CI_PYTHON_PATH": "" // Update with path to real python interpereter used for testing.
+                }
+            }
+        },
+        {
+            "label": "Start Jupyter Server Task",
+            "command": "echo ${input:terminateJupyterServerTask}",
+            "type": "shell",
+            "problemMatcher": []
+        },
+        {
+            "label": "Stop Jupyter Server",
+            "type": "npm",
+            "script": "stopJupyterServer",
+            "problemMatcher": []
+        }
+    ],
+    "inputs": [
+        {
+            "id": "terminateJupyterServerTask",
+            "type": "command",
+            "command": "workbench.action.tasks.terminate",
+            "args": "terminateAll"
         }
     ]
 }

build/launchWebTest.js

@@ -3,28 +3,14 @@
 
 const path = require('path');
 const test_web = require('@vscode/test-web');
-const jupyterServer = require('../out/test/datascience/jupyterServer.node');
-const fs = require('fs-extra');
-
+const { startJupyter } = require('./preLaunchWebTest');
 async function go() {
     let exitCode = 0;
-    const server = jupyterServer.JupyterServer.instance;
+    let server;
     try {
-        // Need to start jupyter here before starting the test as it requires node to start it.
-        const uri = await server.startJupyterWithToken();
-
-        // Use this token to write to the bundle so we can transfer this into the test.
+        server = (await startJupyter()).server;
         const extensionDevelopmentPath = path.resolve(__dirname, '../');
         const bundlePath = path.join(extensionDevelopmentPath, 'out', 'extension.web.bundle');
-        const bundleFile = `${bundlePath}.js`;
-        if (await fs.pathExists(bundleFile)) {
-            const bundleContents = await fs.readFile(bundleFile, { encoding: 'utf-8' });
-            const newContents = bundleContents.replace(
-                /^exports\.JUPYTER_SERVER_URI = '(.*)';$/gm,
-                `exports.JUPYTER_SERVER_URI = '${uri.toString()}';`
-            );
-            await fs.writeFile(bundleFile, newContents);
-        }
 
         // Now run the test
         await test_web.runTests({
@@ -36,9 +22,10 @@ async function go() {
             extensionTestsPath: bundlePath
         });
     } catch (err) {
-        console.error('Failed to run tests');
+        console.error('Failed to run tests', err);
         exitCode = 1;
     } finally {
+        console.error(server);
         if (server) {
             await server.dispose();
         }

build/postDebugWebTest.js

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+const fs = require('fs-extra');
+const path = require('path');
+
+try {
+    const file = path.join(__dirname, '..', 'temp', 'jupyter.pid');
+    if (fs.existsSync(file)) {
+        const pid = parseInt(fs.readFileSync(file).toString().trim());
+        fs.unlinkSync(file);
+        if (pid > 0) {
+            process.kill(pid);
+        }
+    }
+} catch (ex) {
+    console.warn(`Failed to kill Jupyter Server`, ex);
+}

build/preDebugWebTest.js

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+const fs = require('fs-extra');
+const path = require('path');
+const { startJupyter } = require('./preLaunchWebTest');
+const jsonc = require('jsonc-parser');
+
+const settingsFile = path.join(__dirname, '..', 'src', 'test', 'datascience', '.vscode', 'settings.json');
+async function go() {
+    const { server, url } = await startJupyter(true);
+    fs.writeFileSync(path.join(__dirname, '..', 'temp', 'jupyter.pid'), server.pid.toString());
+    const settingsJson = fs.readFileSync(settingsFile).toString();
+    const edits = jsonc.modify(settingsJson, ['jupyter.DEBUG_JUPYTER_SERVER_URI'], url, {});
+    const updatedSettingsJson = jsonc.applyEdits(settingsJson, edits);
+    fs.writeFileSync(settingsFile, updatedSettingsJson);
+    process.exit(0);
+}
+void go();

build/preLaunchWebTest.js

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+const path = require('path');
+const jupyterServer = require('../out/test/datascience/jupyterServer.node');
+const fs = require('fs-extra');
+
+exports.startJupyter = async function startJupyter(detached) {
+    const server = jupyterServer.JupyterServer.instance;
+    // Need to start jupyter here before starting the test as it requires node to start it.
+    const uri = await server.startJupyterWithToken({ detached });
+
+    // Use this token to write to the bundle so we can transfer this into the test.
+    const extensionDevelopmentPath = path.resolve(__dirname, '../');
+    const bundlePath = path.join(extensionDevelopmentPath, 'out', 'extension.web.bundle');
+    const bundleFile = `${bundlePath}.js`;
+    if (await fs.pathExists(bundleFile)) {
+        const bundleContents = await fs.readFile(bundleFile, { encoding: 'utf-8' });
+        const newContents = bundleContents.replace(
+            /^exports\.JUPYTER_SERVER_URI = '(.*)';$/gm,
+            `exports.JUPYTER_SERVER_URI = '${uri.toString()}';`
+        );
+        if (newContents === bundleContents) {
+            throw new Error('JUPYTER_SERVER_URI in bundle not updated');
+        }
+        await fs.writeFile(bundleFile, newContents);
+    }
+    return { server, url: uri.toString() };
+};

package.json

@@ -2142,6 +2142,8 @@
         "postdownload-api": "vscode-dts main",
         "generateTelemetry": "gulp generateTelemetryMd",
         "openInBrowser": "vscode-test-web --extensionDevelopmentPath=. ./src/test/datascience",
+        "startJupyterServer": "node build/preDebugWebTest.js",
+        "stopJupyterServer": "node build/postDebugWebTest.js",
         "validateTelemetryMD": "gulp validateTelemetryMD",
         "prepare": "husky install"
     },

src/test/common.web.ts

@@ -35,9 +35,11 @@ export function initializeCommonWebApi() {
             };
         },
         async startJupyterServer(notebook?: NotebookDocument): Promise<void> {
+            // DEBUG_JUPYTER_SERVER_URI is not a valid setting, but updated when we launch the tests via vscode debugger.
+            const url = workspace.getConfiguration('jupyter').get('DEBUG_JUPYTER_SERVER_URI', JUPYTER_SERVER_URI);
+            console.log(`ServerURI for remote test: ${url}`);
             // Server URI should have been embedded in the constants file
-            const uri = Uri.parse(JUPYTER_SERVER_URI);
-            console.log(`ServerURI for remote test: ${JUPYTER_SERVER_URI}`);
+            const uri = Uri.parse(url);
             // Use this URI to set our jupyter server URI
             await commands.executeCommand('jupyter.selectjupyteruri', false, uri, notebook);
         },

src/test/datascience/.vscode/settings.json

@@ -32,5 +32,9 @@
     "jupyter.generateSVGPlots": false,
     // See https://github.com/microsoft/vscode-jupyter/issues/10258
     "jupyter.forceIPyKernelDebugger": false,
-    "python.defaultInterpreterPath": "python"
+    "python.defaultInterpreterPath": "python",
+    "task.problemMatchers.neverPrompt": {
+        "shell": true,
+        "npm": true
+    }
 }

src/test/datascience/jupyterServer.node.ts

@@ -71,6 +71,10 @@ function traceInfoIfCI(message: string) {
 }
 
 export class JupyterServer {
+    /**
+     * Used in vscode debugger launcher `preDebugWebTest.js` to kill the Jupyter Server by pid.
+     */
+    public pid: number = -1;
     public static get instance(): JupyterServer {
         if (!JupyterServer._instance) {
             JupyterServer._instance = new JupyterServer();
@@ -100,7 +104,7 @@ export class JupyterServer {
         }
     }
 
-    public async startJupyterWithCert(): Promise<string> {
+    public async startJupyterWithCert(detached?: boolean): Promise<string> {
         if (!this._jupyterServerWithCert) {
             this._jupyterServerWithCert = new Promise<string>(async (resolve, reject) => {
                 const token = this.generateToken();
@@ -112,7 +116,8 @@ export class JupyterServer {
                     await this.startJupyterServer({
                         port,
                         token,
-                        useCert: true
+                        useCert: true,
+                        detached
                     });
                     await sleep(5_000); // Wait for some time for Jupyter to warm up & be ready to accept connections.
 
@@ -126,7 +131,8 @@ export class JupyterServer {
         return this._jupyterServerWithCert;
     }
 
-    public async startJupyterWithToken(token = this.generateToken()): Promise<string> {
+    public async startJupyterWithToken({ detached }: { detached?: boolean } = {}): Promise<string> {
+        const token = this.generateToken();
         if (!this._jupyterServerWithToken) {
             this._jupyterServerWithToken = new Promise<string>(async (resolve, reject) => {
                 const port = await this.getFreePort();
@@ -136,10 +142,13 @@ export class JupyterServer {
                 try {
                     await this.startJupyterServer({
                         port,
-                        token
+                        token,
+                        detached
                     });
                     await sleep(5_000); // Wait for some time for Jupyter to warm up & be ready to accept connections.
-                    resolve(`http://localhost:${port}/?token=${token}`);
+                    const url = `http://localhost:${port}/?token=${token}`;
+                    console.log(`Started Jupyter Server on ${url}`);
+                    resolve(url);
                 } catch (ex) {
                     reject(ex);
                 }
@@ -160,7 +169,9 @@ export class JupyterServer {
                         token
                     });
                     await sleep(5_000); // Wait for some time for Jupyter to warm up & be ready to accept connections.
-                    resolve(`http://localhost:${port}/?token=${token}`);
+                    const url = `http://localhost:${port}/?token=${token}`;
+                    console.log(`Started Jupyter Server on ${url}`);
+                    resolve(url);
                 } catch (ex) {
                     reject(ex);
                 }
@@ -201,11 +212,13 @@ export class JupyterServer {
     private startJupyterServer({
         token,
         port,
-        useCert
+        useCert,
+        detached
     }: {
         token: string;
         port: number;
         useCert?: boolean;
+        detached?: boolean;
     }): Promise<void> {
         return new Promise<void>(async (resolve, reject) => {
             try {
@@ -240,11 +253,13 @@ export class JupyterServer {
                 }
                 traceInfoIfCI(`Starting Jupyter on CI with args ${args.join(' ')}`);
                 const result = this.execObservable(getPythonPath(), args, {
-                    cwd: testFolder
+                    cwd: testFolder,
+                    detached
                 });
                 if (!result.proc) {
                     throw new Error('Starting Jupyter failed, no process');
                 }
+                this.pid = result.proc.pid;
                 result.proc.once('close', () => traceInfo('Shutting Jupyter server used for remote tests (closed)'));
                 result.proc.once('disconnect', () =>
                     traceInfo('Shutting Jupyter server used for remote tests (disconnected)')
@@ -263,6 +278,9 @@ export class JupyterServer {
                     }
                 };
                 const subscription = result.out.subscribe((output) => {
+                    // When debugging Web Tests using VSCode dfebugger, we'd like to see this info.
+                    // This way we can click the link in the output panel easily.
+                    console.info(output.out);
                     traceInfoIfCI(`Test Remote Jupyter Server Output: ${output.out}`);
                     if (output.out.indexOf('Use Control-C to stop this server and shut down all kernels')) {
                         resolve();