generalize code and markdown code lenses to be used with multiple languages

.eslintrc.js

@@ -179,7 +179,6 @@ module.exports = {
         'src/test/datascience/mockCustomEditorService.ts',
         'src/test/datascience/mockInputBox.ts',
         'src/test/datascience/mockDocumentManager.ts',
-        'src/test/datascience/cellMatcher.unit.test.ts',
         'src/test/datascience/crossProcessLock.unit.test.ts',
         'src/test/datascience/uiTests/helpers.ts',
         'src/test/datascience/uiTests/notebookUi.ts',
@@ -260,7 +259,6 @@ module.exports = {
         'src/test/datascience/jupyter/serverCache.unit.test.ts',
         'src/test/datascience/mockWorkspaceConfig.ts',
         'src/test/datascience/mockProcessService.ts',
-        'src/test/datascience/cellFactory.unit.test.ts',
         'src/test/refactor/extension.refactor.extract.method.test.ts',
         'src/test/refactor/extension.refactor.extract.var.test.ts',
         'src/test/refactor/rename.test.ts',
@@ -287,7 +285,6 @@ module.exports = {
         'src/datascience-ui/native-editor/redux/mapping.ts',
         'src/datascience-ui/native-editor/redux/reducers/movement.ts',
         'src/datascience-ui/native-editor/redux/reducers/creation.ts',
-        'src/datascience-ui/native-editor/redux/reducers/execution.ts',
         'src/datascience-ui/native-editor/redux/reducers/effects.ts',
         'src/datascience-ui/renderers/index.tsx',
         'src/datascience-ui/renderers/webviewApi.d.ts',
@@ -311,7 +308,6 @@ module.exports = {
         'src/datascience-ui/interactive-common/buildSettingsCss.ts',
         'src/datascience-ui/interactive-common/markdown.tsx',
         'src/datascience-ui/interactive-common/editor.tsx',
-        'src/datascience-ui/interactive-common/mainState.ts',
         'src/datascience-ui/interactive-common/collapseButton.tsx',
         'src/datascience-ui/interactive-common/utils.ts',
         'src/datascience-ui/interactive-common/images.d.ts',
@@ -338,7 +334,6 @@ module.exports = {
         'src/datascience-ui/history-react/redux/mapping.ts',
         'src/datascience-ui/history-react/redux/reducers/index.ts',
         'src/datascience-ui/history-react/redux/reducers/creation.ts',
-        'src/datascience-ui/history-react/redux/reducers/execution.ts',
         'src/datascience-ui/history-react/redux/reducers/effects.ts',
         'src/datascience-ui/react-common/arePathsSame.ts',
         'src/datascience-ui/react-common/imageButton.tsx',
@@ -801,7 +796,6 @@ module.exports = {
         'src/client/datascience/themeFinder.ts',
         'src/client/datascience/multiplexingDebugService.ts',
         'src/client/datascience/interactive-window/identity.ts',
-        'src/client/datascience/datascience.ts',
         'src/client/datascience/liveshare/liveshare.ts',
         'src/client/datascience/liveshare/serviceProxy.ts',
         'src/client/datascience/liveshare/liveshareProxy.ts',
@@ -823,7 +817,6 @@ module.exports = {
         'src/client/datascience/interactive-common/intellisense/wordHelper.ts',
         'src/client/datascience/interactive-common/intellisense/intellisenseLine.ts',
         'src/client/datascience/jupyterUriProviderWrapper.ts',
-        'src/client/datascience/cellMatcher.ts',
         'src/client/datascience/notebookStorage/notebookModelEditEvent.ts',
         'src/client/datascience/debugLocationTracker.ts',
         'src/client/datascience/plotting/plotViewerMessageListener.ts',
@@ -836,9 +829,7 @@ module.exports = {
         'src/client/datascience/interactive-ipynb/nativeEditorSynchronizer.ts',
         'src/client/datascience/interactive-ipynb/autoSaveService.ts',
         'src/client/datascience/editor-integration/decorator.ts',
-        'src/client/datascience/editor-integration/cellhashprovider.ts',
         'src/client/datascience/commands/commandLineSelector.ts',
-        'src/client/datascience/cellFactory.ts',
         'src/client/datascience/notebook/notebookEditorCompatibilitySupport.ts',
         'src/client/datascience/notebook/constants.ts',
         'src/client/datascience/notebook/renderer.ts',
@@ -872,7 +863,7 @@ module.exports = {
         'src/client/datascience/jupyter/jupyterPasswordConnect.ts',
         'src/client/datascience/jupyter/jupyterWebSocket.ts',
         'src/client/datascience/jupyter/variableScriptLoader.ts',
-        'src/client/datascience/jupyter/jupyterImporter.ts',
+        'src/client/datascience/jupyter/jupyterInstallError.ts',
         'src/client/datascience/jupyter/oldJupyterVariables.ts',
         'src/client/datascience/dataScienceFileSystem.ts',
         'src/client/logging/levels.ts',

package.json

@@ -44,6 +44,7 @@
     ],
     "activationEvents": [
         "onLanguage:python",
+        "onLanguage:markdown",
         "onLanguage:jupyter",
         "onCommand:jupyter.viewOutput",
         "onCommand:jupyter.export",
@@ -80,7 +81,7 @@
             {
                 "command": "jupyter.execSelectionInteractive",
                 "key": "shift+enter",
-                "when": "editorTextFocus && editorLangId == python && !findInputFocussed && !replaceInputFocussed && jupyter.ownsSelection && !notebookEditorFocused"
+                "when": "editorTextFocus &&  !findInputFocussed && !replaceInputFocussed && jupyter.ownsSelection && !notebookEditorFocused"
             },
             {
                 "command": "jupyter.runcurrentcelladvance",
@@ -754,52 +755,52 @@
         "menus": {
             "editor/context": [
                 {
-                    "when": "editorFocus && editorLangId == python && jupyter.hascodecells && !notebookEditorFocused && isWorkspaceTrusted",
+                    "when": "editorFocus && jupyter.hascodecells && !notebookEditorFocused && isWorkspaceTrusted",
                     "command": "jupyter.runallcells",
                     "group": "Jupyter2"
                 },
                 {
-                    "when": "editorFocus && editorLangId == python && jupyter.hascodecells && !notebookEditorFocused && isWorkspaceTrusted",
+                    "when": "editorFocus && jupyter.hascodecells && !notebookEditorFocused && isWorkspaceTrusted",
                     "command": "jupyter.runcurrentcell",
                     "group": "Jupyter2"
                 },
                 {
-                    "when": "editorFocus && editorLangId == python && jupyter.hascodecells && !notebookEditorFocused && isWorkspaceTrusted",
+                    "when": "editorFocus && jupyter.hascodecells && !notebookEditorFocused && isWorkspaceTrusted",
                     "command": "jupyter.runcurrentcelladvance",
                     "group": "Jupyter2"
                 },
                 {
                     "command": "jupyter.runFileInteractive",
                     "group": "Jupyter2",
-                    "when": "editorFocus && editorLangId == python && !notebookEditorFocused && isWorkspaceTrusted"
+                    "when": "editorFocus && !notebookEditorFocused && isWorkspaceTrusted"
                 },
                 {
                     "command": "jupyter.runfromline",
                     "group": "Jupyter2",
-                    "when": "editorFocus && editorLangId == python && !notebookEditorFocused && isWorkspaceTrusted"
+                    "when": "editorFocus && !notebookEditorFocused && isWorkspaceTrusted"
                 },
                 {
                     "command": "jupyter.runtoline",
                     "group": "Jupyter2",
-                    "when": "editorFocus && editorLangId == python && !notebookEditorFocused && isWorkspaceTrusted"
+                    "when": "editorFocus && && !notebookEditorFocused && isWorkspaceTrusted"
                 },
                 {
                     "command": "jupyter.execSelectionInteractive",
                     "group": "Jupyter2",
-                    "when": "editorFocus && editorLangId == python && !notebookEditorFocused && isWorkspaceTrusted"
+                    "when": "editorFocus && !notebookEditorFocused && isWorkspaceTrusted"
                 },
                 {
                     "when": "editorFocus && editorLangId == python && resourceLangId == jupyter && !notebookEditorFocused && isWorkspaceTrusted",
                     "command": "jupyter.importnotebook",
                     "group": "Jupyter3@1"
                 },
                 {
-                    "when": "editorFocus && editorLangId == python && jupyter.hascodecells && !notebookEditorFocused && isWorkspaceTrusted",
+                    "when": "editorFocus && jupyter.hascodecells && !notebookEditorFocused && isWorkspaceTrusted",
                     "command": "jupyter.exportfileasnotebook",
                     "group": "Jupyter3@2"
                 },
                 {
-                    "when": "editorFocus && editorLangId == python && jupyter.hascodecells && !notebookEditorFocused && isWorkspaceTrusted",
+                    "when": "editorFocus && jupyter.hascodecells && !notebookEditorFocused && isWorkspaceTrusted",
                     "command": "jupyter.exportfileandoutputasnotebook",
                     "group": "Jupyter3@3"
                 }
@@ -1026,7 +1027,7 @@
                     "command": "jupyter.debugcurrentcell.palette",
                     "title": "%jupyter.command.jupyter.debugcurrentcell.palette.title%",
                     "category": "Jupyter",
-                    "when": "editorLangId == python && jupyter.hascodecells && isWorkspaceTrusted"
+                    "when": "jupyter.hascodecells && isWorkspaceTrusted"
                 },
                 {
                     "command": "jupyter.createnewinteractive",
@@ -1062,13 +1063,13 @@
                     "command": "jupyter.runFileInteractive",
                     "title": "%jupyter.command.jupyter.runFileInteractive.title%",
                     "category": "Jupyter",
-                    "when": "editorLangId == python && !notebookEditorFocused && isWorkspaceTrusted"
+                    "when": "!notebookEditorFocused && isWorkspaceTrusted"
                 },
                 {
                     "command": "jupyter.debugFileInteractive",
                     "title": "%jupyter.command.jupyter.debugFileInteractive.title%",
                     "category": "Jupyter",
-                    "when": "editorLangId == python && !notebookEditorFocused && isWorkspaceTrusted"
+                    "when": "!notebookEditorFocused && isWorkspaceTrusted"
                 },
                 {
                     "command": "jupyter.importnotebook",
@@ -1256,7 +1257,7 @@
                 {
                     "command": "jupyter.execSelectionInteractive",
                     "category": "Jupyter",
-                    "when": "editorLangId == python && !notebookEditorFocused && isWorkspaceTrusted"
+                    "when": "!notebookEditorFocused && isWorkspaceTrusted"
                 },
                 {
                     "command": "jupyter.switchKernel",
@@ -1548,22 +1549,44 @@
                 },
                 "jupyter.codeRegularExpression": {
                     "type": "string",
+                    "deprecationMessage": "This setting is no longer being used. Use jupyter.codeLensExpressions instead",
                     "default": "^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
                     "description": "Regular expression used to identify code cells. All code until the next match is considered part of this cell.",
                     "scope": "resource"
                 },
                 "jupyter.defaultCellMarker": {
                     "type": "string",
+                    "deprecationMessage": "This setting is no longer being used. Use jupyter.codeLensExpressions instead",
                     "default": "# %%",
                     "description": "Cell marker used for delineating a cell in a python file.",
                     "scope": "resource"
                 },
                 "jupyter.markdownRegularExpression": {
                     "type": "string",
+                    "deprecationMessage": "This setting is no longer being used. Use jupyter.codeLensExpressions instead",
                     "default": "^(#\\s*%%\\s*\\[markdown\\]|#\\s*\\<markdowncell\\>)",
                     "description": "Regular expression used to identify markdown cells. All comments after this expression are considered part of the markdown.",
                     "scope": "resource"
                 },
+                "jupyter.codeLensExpressions": {
+                    "type": "array",
+                    "description": "Language to code lens mapping for identifying markdown and code cells based on different languages or file extensions",
+                    "scope": "machine",
+                    "default": [
+                        {
+                            "language": "python",
+                            "codeExpression": "^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
+                            "markdownExpression": "^(#\\s*%%\\s*\\[markdown\\]|#\\s*\\<markdowncell\\>)",
+                            "defaultCellMarker": "# %%"
+                        },
+                        {
+                            "language": "markdown",
+                            "codeExpression": "^(```python\\s*|```\\{code-cell\\}\\s+ipython)",
+                            "markdownExpression": "(?!x)x",
+                            "defaultCellMarker": "```python"
+                        }
+                    ]
+                },
                 "jupyter.allowLiveShare": {
                     "type": "boolean",
                     "default": true,

src/client/common/configSettings.ts

@@ -15,6 +15,7 @@ import { IWorkspaceService } from './application/types';
 import { WorkspaceService } from './application/workspace';
 import { isTestExecution } from './constants';
 import {
+    ICodeLensExpressions,
     IExperiments,
     ILoggingSettings,
     InteractiveWindowMode,
@@ -84,6 +85,7 @@ export class JupyterSettings implements IWatchableJupyterSettings {
     public defaultCellMarker: string = '';
     public themeMatplotlibPlots: boolean = false;
     public variableQueries: IVariableQuery[] = [];
+    public codeLensExpressions: ICodeLensExpressions[] = [];
     public disableJupyterAutoStart: boolean = false;
     public jupyterCommandLineArguments: string[] = [];
     public widgetScriptSources: WidgetCDNs[] = [];

src/client/common/types.ts

@@ -132,9 +132,9 @@ export interface IJupyterSettings {
     readonly maxOutputSize: number;
     readonly enableScrollingForCellOutputs: boolean;
     readonly sendSelectionToInteractiveWindow: boolean;
-    readonly markdownRegularExpression: string;
-    readonly codeRegularExpression: string;
     readonly allowLiveShare: boolean;
+    readonly markdownRegularExpression: string | undefined;
+    readonly codeRegularExpression: string | undefined;
     readonly errorBackgroundColor: string;
     readonly ignoreVscodeTheme: boolean;
     readonly variableExplorerExclude: string;
@@ -158,10 +158,11 @@ export interface IJupyterSettings {
     readonly runMagicCommands: string;
     readonly runStartupCommands: string | string[];
     readonly debugJustMyCode: boolean;
-    readonly defaultCellMarker: string;
     readonly verboseLogging: boolean;
+    readonly defaultCellMarker: string | undefined;
     readonly themeMatplotlibPlots: boolean;
     readonly variableQueries: IVariableQuery[];
+    readonly codeLensExpressions: ICodeLensExpressions[];
     readonly disableJupyterAutoStart: boolean;
     readonly jupyterCommandLineArguments: string[];
     readonly widgetScriptSources: WidgetCDNs[];
@@ -208,6 +209,13 @@ export interface IVariableQuery {
     parseExpr: string;
 }
 
+export interface ICodeLensExpressions {
+    language: string;
+    codeExpression: string;
+    markdownExpression: string;
+    defaultCellMarker: string;
+}
+
 export type InteractiveWindowMode = 'perFile' | 'single' | 'multiple';
 
 export type WidgetCDNs = 'unpkg.com' | 'jsdelivr.com';

src/client/datascience/cellFactory.ts

@@ -54,12 +54,13 @@ export function generateCells(
     file: string,
     line: number,
     splitMarkdown: boolean,
-    id: string
+    id: string,
+    language: string | undefined
 ): ICell[] {
     // Determine if we have a markdown cell/ markdown and code cell combined/ or just a code cell
     const split = code.splitLines({ trim: false });
     const firstLine = split[0];
-    const matcher = new CellMatcher(settings);
+    const matcher = new CellMatcher(language, settings);
     const { magicCommandsAsComments = false } = settings || {};
     if (matcher.isMarkdown(firstLine)) {
         // We have at least one markdown. We might have to split it if there any lines that don't begin
@@ -97,8 +98,8 @@ export function generateCells(
     }
 }
 
-export function hasCells(document: TextDocument, settings?: IJupyterSettings): boolean {
-    const matcher = new CellMatcher(settings);
+export function hasCells(document: TextDocument, settings?: IJupyterSettings, language?: string): boolean {
+    const matcher = new CellMatcher(language, settings);
     for (let index = 0; index < document.lineCount; index += 1) {
         const line = document.lineAt(index);
         if (matcher.isCell(line.text)) {
@@ -109,11 +110,11 @@ export function hasCells(document: TextDocument, settings?: IJupyterSettings): b
     return false;
 }
 
-export function generateCellsFromString(source: string, settings?: IJupyterSettings): ICell[] {
+export function generateCellsFromString(source: string, settings?: IJupyterSettings, language?: string): ICell[] {
     const lines: string[] = source.splitLines({ trim: false, removeEmptyEntries: false });
 
     // Find all the lines that start a cell
-    const matcher = new CellMatcher(settings);
+    const matcher = new CellMatcher(language, settings);
     const starts: { startLine: number; title: string; code: string; cell_type: string }[] = [];
     let currentCode: string | undefined;
     for (let index = 0; index < lines.length; index += 1) {
@@ -145,14 +146,18 @@ export function generateCellsFromString(source: string, settings?: IJupyterSetti
     // For each one, get its text and turn it into a cell
     return Array.prototype.concat(
         ...starts.map((s) => {
-            return generateCells(settings, s.code, '', s.startLine, false, uuid());
+            return generateCells(settings, s.code, '', s.startLine, false, uuid(), language);
         })
     );
 }
 
-export function generateCellRangesFromDocument(document: TextDocument, settings?: IJupyterSettings): ICellRange[] {
+export function generateCellRangesFromDocument(
+    document: TextDocument,
+    settings?: IJupyterSettings,
+    language?: string
+): ICellRange[] {
     // Implmentation of getCells here based on Don's Jupyter extension work
-    const matcher = new CellMatcher(settings);
+    const matcher = new CellMatcher(language, settings);
     const cells: ICellRange[] = [];
     for (let index = 0; index < document.lineCount; index += 1) {
         const line = document.lineAt(index);
@@ -189,7 +194,7 @@ export function generateCellsFromDocument(document: TextDocument, settings?: IJu
     return Array.prototype.concat(
         ...ranges.map((cr) => {
             const code = document.getText(cr.range);
-            return generateCells(settings, code, '', cr.range.start.line, false, uuid());
+            return generateCells(settings, code, '', cr.range.start.line, false, uuid(), document.languageId);
         })
     );
 }