Load widget scripts from all known Jupyter paths

.github/workflows/build-test.yml

@@ -481,6 +481,22 @@ jobs:
           beakerx_kernel_java install
           conda install pytorch cpuonly -c pytorch
 
+      - name: Install pythreejs and matplotlib widgets into user and system paths
+        if: matrix.os == 'ubuntu-latest' && matrix.python != 'conda' && matrix.python != 'noPython'
+        # This test will ensure widgets work when installed in 3 places
+        # 1. In python environments site-packages folder (we have other 3rd party widgets in the python env)
+        # 2. In user's home folder (pythreejs will be installed in there)
+        # 3. In system folder (all users) (matplotlib will be installed in there)
+        run: |
+            # Uninstall pythreejs from the sys prefix folder and ensure the widget scripts are installed
+            # into the user directory.
+            export PYTHON_EXECUTABLE=$(which python)
+            echo $PYTHON_EXECUTABLE
+            python -m jupyter nbextension uninstall --sys-prefix --py pythreejs
+            python -m jupyter nbextension install --user --py pythreejs
+            python -m jupyter nbextension uninstall --sys-prefix --py ipympl
+            sudo $PYTHON_EXECUTABLE -m jupyter nbextension install --system --py ipympl
+
         # This step is slow.
       - name: Install dependencies (npm ci)
         run: npm ci --prefer-offline

build/venv-test-requirements.txt

@@ -23,3 +23,6 @@ ipyleaflet
 jinja2==3.0.3 # https://github.com/microsoft/vscode-jupyter/issues/9468#issuecomment-1078468039
 matplotlib
 ipympl
+ase
+chemiscope
+mobilechelonian

pythonFiles/printJupyWidgetEntryPoints.py

@@ -0,0 +1,215 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+# Source borrowed from site-packages/jupyter_core/paths.py
+
+
+def __vsc_print_nbextension_widgets():
+    import os as __vscode_os
+    import site as __vscode_site
+    import sys as __vscode_sys
+    import tempfile as __vscode_tempfile
+    from pathlib import Path as __vscode_Path
+
+    pjoin = __vscode_os.path.join
+
+    def envset(name):
+        """Return True if the given environment variable is set
+
+        An environment variable is considered set if it is assigned to a value
+        other than 'no', 'n', 'false', 'off', '0', or '0.0' (case insensitive)
+        """
+        return __vscode_os.environ.get(name, "no").lower() not in [
+            "no",
+            "n",
+            "false",
+            "off",
+            "0",
+            "0.0",
+        ]
+
+    def get_home_dir():
+        """Get the real path of the home directory"""
+        homedir = __vscode_os.path.expanduser("~")
+        # Next line will make things work even when /home/ is a symlink to
+        # /usr/home as it is on FreeBSD, for example
+        homedir = str(__vscode_Path(homedir).resolve())
+        return homedir
+
+    _dtemps: dict = {}
+
+    def _mkdtemp_once(name):
+        """Make or reuse a temporary directory.
+
+        If this is called with the same name in the same process, it will return
+        the same directory.
+        """
+        try:
+            return _dtemps[name]
+        except KeyError:
+            d = _dtemps[name] = __vscode_tempfile.mkdtemp(prefix=name + "-")
+            return d
+
+    def jupyter_config_dir():
+        """Get the Jupyter config directory for this platform and user.
+
+        Returns JUPYTER_CONFIG_DIR if defined, else ~/.jupyter
+        """
+
+        env = __vscode_os.environ
+        if env.get("JUPYTER_NO_CONFIG"):
+            return _mkdtemp_once("jupyter-clean-cfg")
+
+        if env.get("JUPYTER_CONFIG_DIR"):
+            return env["JUPYTER_CONFIG_DIR"]
+
+        home_dir = get_home_dir()
+        return pjoin(home_dir, ".jupyter")
+
+    def jupyter_data_dir():
+        """Get the config directory for Jupyter data files for this platform and user.
+
+        These are non-transient, non-configuration files.
+
+        Returns JUPYTER_DATA_DIR if defined, else a platform-appropriate path.
+        """
+        env = __vscode_os.environ
+
+        if env.get("JUPYTER_DATA_DIR"):
+            return env["JUPYTER_DATA_DIR"]
+
+        home = get_home_dir()
+
+        if __vscode_sys.platform == "darwin":
+            return __vscode_os.path.join(home, "Library", "Jupyter")
+        elif __vscode_os.name == "nt":
+            appdata = __vscode_os.environ.get("APPDATA", None)
+            if appdata:
+                return str(__vscode_Path(appdata, "jupyter").resolve())
+            else:
+                return pjoin(jupyter_config_dir(), "data")
+        else:
+            # Linux, non-OS X Unix, AIX, etc.
+            xdg = env.get("XDG_DATA_HOME", None)
+            if not xdg:
+                xdg = pjoin(home, ".local", "share")
+            return pjoin(xdg, "jupyter")
+
+    if __vscode_os.name == "nt":
+        programdata = __vscode_os.environ.get("PROGRAMDATA", None)
+        if programdata:
+            SYSTEM_JUPYTER_PATH = [pjoin(programdata, "jupyter")]
+        else:  # PROGRAMDATA is not defined by default on XP.
+            SYSTEM_JUPYTER_PATH = [
+                __vscode_os.path.join(__vscode_sys.prefix, "share", "jupyter")
+            ]
+    else:
+        SYSTEM_JUPYTER_PATH = [
+            "/usr/local/share/jupyter",
+            "/usr/share/jupyter",
+        ]
+
+    ENV_JUPYTER_PATH = [__vscode_os.path.join(__vscode_sys.prefix, "share", "jupyter")]
+
+    def jupyter_path(*subdirs):
+        """Return a list of directories to search for data files
+
+        JUPYTER_PATH environment variable has highest priority.
+
+        If the JUPYTER_PREFER_ENV_PATH environment variable is set, the environment-level
+        directories will have priority over user-level directories.
+
+        If the Python __vscode_site.ENABLE_USER_SITE variable is True, we also add the
+        appropriate Python user site subdirectory to the user-level directories.
+
+
+        If ``*subdirs`` are given, that subdirectory will be added to each element.
+
+        Examples:
+
+        >>> jupyter_path()
+        ['~/.local/jupyter', '/usr/local/share/jupyter']
+        >>> jupyter_path('kernels')
+        ['~/.local/jupyter/kernels', '/usr/local/share/jupyter/kernels']
+        """
+
+        paths: list = []
+
+        # highest priority is explicit environment variable
+        if __vscode_os.environ.get("JUPYTER_PATH"):
+            paths.extend(
+                p.rstrip(__vscode_os.sep)
+                for p in __vscode_os.environ["JUPYTER_PATH"].split(__vscode_os.pathsep)
+            )
+
+        # Next is environment or user, depending on the JUPYTER_PREFER_ENV_PATH flag
+        user = [jupyter_data_dir()]
+        if __vscode_site.ENABLE_USER_SITE:
+            # Check if __vscode_site.getuserbase() exists to be compatible with virtualenv,
+            # which often does not have this method.
+            if hasattr(__vscode_site, "getuserbase"):
+                userbase = __vscode_site.getuserbase()
+            else:
+                userbase = __vscode_site.USER_BASE
+
+            if userbase:
+                userdir = __vscode_os.path.join(userbase, "share", "jupyter")
+                if userdir not in user:
+                    user.append(userdir)
+
+        env = [p for p in ENV_JUPYTER_PATH if p not in SYSTEM_JUPYTER_PATH]
+
+        if envset("JUPYTER_PREFER_ENV_PATH"):
+            paths.extend(env)
+            paths.extend(user)
+        else:
+            paths.extend(user)
+            paths.extend(env)
+
+        # finally, system
+        paths.extend(SYSTEM_JUPYTER_PATH)
+
+        # add subdir, if requested
+        if subdirs:
+            paths = [pjoin(p, *subdirs) for p in paths]
+        return paths
+
+    __vsc_nbextension_widgets = []
+    __vsc_file = ""
+    __vsc_nbextension_Folder = ""
+    __vscode_widget_folder = ""
+    import glob as _VSCODE_glob
+
+    try:
+        for __vsc_nbextension_Folder in jupyter_path("nbextensions"):
+            for __vsc_file in _VSCODE_glob.glob(
+                __vsc_nbextension_Folder
+                + __vscode_os.path.sep
+                + "*"
+                + __vscode_os.path.sep
+                + "extension.js"
+            ):
+                __vscode_widget_folder = __vsc_file.replace(
+                    __vsc_nbextension_Folder, ""
+                )
+                if not __vscode_widget_folder in __vsc_nbextension_widgets:
+                    __vsc_nbextension_widgets.append(__vscode_widget_folder)
+
+        print(__vsc_nbextension_widgets)
+    except:
+        pass
+
+    # We need to ensure these variables don't interfere with the variable viewer, hence delete them after use.
+    del _VSCODE_glob
+    del __vsc_file
+    del __vsc_nbextension_Folder
+    del __vscode_widget_folder
+    del __vsc_nbextension_widgets
+    del __vscode_os
+    del __vscode_site
+    del __vscode_sys
+    del __vscode_tempfile
+    del __vscode_Path
+
+
+__vsc_print_nbextension_widgets()

pythonFiles/printJupyterDataDir.py

@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import os
+import site
+
+# Copied from site-packages/jupyter_core/paths.py
+
+if site.ENABLE_USER_SITE:
+    # Check if site.getuserbase() exists to be compatible with virtualenv,
+    # which often does not have this method.
+    userbase: Optional[str]
+    if hasattr(site, "getuserbase"):
+        userbase = site.getuserbase()
+    else:
+        userbase = site.USER_BASE
+
+    if userbase:
+        userdir = os.path.join(userbase, "share", "jupyter")
+        print(userdir)

src/kernels/ipywidgets/baseIPyWidgetScriptManager.ts

@@ -115,9 +115,6 @@ export function extractRequireConfigFromWidgetEntry(baseUrl: Uri, widgetFolderNa
 
 export abstract class BaseIPyWidgetScriptManager implements IIPyWidgetScriptManager {
     protected readonly disposables: IDisposable[] = [];
-    protected abstract getWidgetEntryPoints(): Promise<{ uri: Uri; widgetFolderName: string }[]>;
-    protected abstract getWidgetScriptSource(source: Uri): Promise<string>;
-    protected abstract getNbExtensionsParentPath(): Promise<Uri | undefined>;
     private widgetModuleMappings?: Promise<Record<string, Uri> | undefined>;
     constructor(protected readonly kernel: IKernel) {
         // If user installs new python packages, & they restart the kernel, then look for changes to nbextensions folder once again.
@@ -129,6 +126,9 @@ export abstract class BaseIPyWidgetScriptManager implements IIPyWidgetScriptMana
         disposeAllDisposables(this.disposables);
     }
     abstract getBaseUrl(): Promise<Uri | undefined>;
+    protected abstract getWidgetEntryPoints(): Promise<{ uri: Uri; widgetFolderName: string }[]>;
+    protected abstract getWidgetScriptSource(source: Uri): Promise<string>;
+    protected abstract getNbExtensionsParentPath(): Promise<Uri | undefined>;
     public async getWidgetModuleMappings(): Promise<Record<string, Uri> | undefined> {
         if (!this.widgetModuleMappings) {
             this.widgetModuleMappings = this.getWidgetModuleMappingsImpl();

src/kernels/ipywidgets/ipyWidgetScriptManagerFactory.node.ts

@@ -27,7 +27,10 @@ export class IPyWidgetScriptManagerFactory implements IIPyWidgetScriptManagerFac
                 kernel.kernelConnectionMetadata.kind === 'connectToLiveRemoteKernel' ||
                 kernel.kernelConnectionMetadata.kind === 'startUsingRemoteKernelSpec'
             ) {
-                this.managers.set(kernel, new RemoteIPyWidgetScriptManager(kernel, this.httpClient, this.context));
+                this.managers.set(
+                    kernel,
+                    new RemoteIPyWidgetScriptManager(kernel, this.httpClient, this.context, this.fs)
+                );
             } else {
                 this.managers.set(
                     kernel,

src/kernels/ipywidgets/ipyWidgetScriptManagerFactory.web.ts

@@ -3,6 +3,7 @@
 // Licensed under the MIT License.
 
 import { injectable, inject } from 'inversify';
+import { IFileSystem } from '../../platform/common/platform/types';
 import { IExtensionContext, IHttpClient } from '../../platform/common/types';
 import { IKernel } from '../types';
 import { RemoteIPyWidgetScriptManager } from './remoteIPyWidgetScriptManager';
@@ -13,15 +14,19 @@ export class IPyWidgetScriptManagerFactory implements IIPyWidgetScriptManagerFac
     private readonly managers = new WeakMap<IKernel, IIPyWidgetScriptManager>();
     constructor(
         @inject(IHttpClient) private readonly httpClient: IHttpClient,
-        @inject(IExtensionContext) private readonly context: IExtensionContext
+        @inject(IExtensionContext) private readonly context: IExtensionContext,
+        @inject(IFileSystem) private readonly fs: IFileSystem
     ) {}
     getOrCreate(kernel: IKernel): IIPyWidgetScriptManager {
         if (!this.managers.has(kernel)) {
             if (
                 kernel.kernelConnectionMetadata.kind === 'connectToLiveRemoteKernel' ||
                 kernel.kernelConnectionMetadata.kind === 'startUsingRemoteKernelSpec'
             ) {
-                this.managers.set(kernel, new RemoteIPyWidgetScriptManager(kernel, this.httpClient, this.context));
+                this.managers.set(
+                    kernel,
+                    new RemoteIPyWidgetScriptManager(kernel, this.httpClient, this.context, this.fs)
+                );
             } else {
                 throw new Error('Cannot enumerate Widget Scripts using local kernels on the Web');
             }

src/kernels/ipywidgets/localIPyWidgetScriptManager.node.ts

@@ -51,6 +51,9 @@ export class LocalIPyWidgetScriptManager extends BaseIPyWidgetScriptManager impl
     }
     protected override onKernelRestarted(): void {
         this.nbExtensionsParentPath = undefined;
+        // Possible there are new versions of nbExtensions that are not yet copied.
+        // E.g. user installs a package and restarts the kernel.
+        this.overwriteExistingFiles = true;
         super.onKernelRestarted();
     }
     protected async getNbExtensionsParentPath(): Promise<Uri | undefined> {
@@ -70,16 +73,25 @@ export class LocalIPyWidgetScriptManager extends BaseIPyWidgetScriptManager impl
             }
             const kernelHash = getTelemetrySafeHashedString(this.kernel.kernelConnectionMetadata.id);
             const baseUrl = Uri.joinPath(this.context.extensionUri, 'tmp', 'scripts', kernelHash, 'jupyter');
+
             const targetNbExtensions = Uri.joinPath(baseUrl, 'nbextensions');
-            const jupyterDataDir = await this.jupyterPaths.getDataDir();
-            const userNbExtensionsDir = jupyterDataDir ? Uri.joinPath(jupyterDataDir, 'nbextensions') : undefined;
-            await this.fs.createDirectory(targetNbExtensions);
-            if (userNbExtensionsDir && (await this.fs.exists(userNbExtensionsDir))) {
-                await this.fs.copy(userNbExtensionsDir, targetNbExtensions, { overwrite });
+            const [jupyterDataDirectories] = await Promise.all([
+                this.jupyterPaths.getDataDirs({
+                    resource: this.kernel.resourceUri,
+                    interpreter: this.kernel.kernelConnectionMetadata.interpreter
+                }),
+                this.fs.createDirectory(targetNbExtensions)
+            ]);
+            const nbExtensionFolders = jupyterDataDirectories.map((dataDir) => Uri.joinPath(dataDir, 'nbextensions'));
+            // The nbextensions folder is sorted in order of priority.
+            // Hence when copying, copy the lowest priority nbextensions folder first.
+            // This way contents get overwritten with contents of highest priority (thereby adhering to the priority).
+            nbExtensionFolders.reverse();
+            for (const nbExtensionFolder of nbExtensionFolders) {
+                if (await this.fs.exists(nbExtensionFolder)) {
+                    await this.fs.copy(nbExtensionFolder, targetNbExtensions, { overwrite });
+                }
             }
-            await this.fs.copy(Uri.joinPath(this.sourceNbExtensionsPath, 'nbextensions'), targetNbExtensions, {
-                overwrite
-            });
             // If we've copied once, then next time, don't overwrite.
             this.overwriteExistingFiles = false;
             LocalIPyWidgetScriptManager.nbExtensionsCopiedKernelConnectionList.add(