From dcf29708f84e5f9e484b6ef843c8f48094036a2c Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Mon, 30 Nov 2020 10:52:56 -0800 Subject: [PATCH 01/11] basic raw idea working with docker container --- .../getVariableInfo/vscodeVariableInfo.py | 51 ++++++++++ src/client/datascience/constants.ts | 18 ++++ .../datascience/jupyter/kernelVariables.ts | 95 ++++++++++++++++++- 3 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeVariableInfo.py diff --git a/pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeVariableInfo.py b/pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeVariableInfo.py new file mode 100644 index 00000000000..7d29bbe4728 --- /dev/null +++ b/pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeVariableInfo.py @@ -0,0 +1,51 @@ +# Query Jupyter server for the info about a dataframe +import json as _VSCODE_json +import builtins as _VSCODE_builtins + +def _VSCODE_maybeParseTensorShape(var, result): + try: + vartype = type(var) + if (hasattr(vartype, "__name__")) and vartype.__name__ == "Tensor": + varshape = str(var.shape) + start = varshape.index("[") + end = varshape.index("]") + if start > 0 and end > 0: + res = "(" + varshape[start + 1 : end] + ")" + result["shape"] = res + except TypeError: + pass + + +# Function to do our work. It will return the object +def _VSCODE_getVariableInfo(var): + # Start out without the information + result = {} + result["shape"] = "" + result["count"] = 0 + + # Find shape and count if available + if hasattr(var, "shape"): + try: + # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it + if isinstance(var.shape, tuple): + _VSCODE_shapeStr = str(var.shape) + if ( + len(_VSCODE_shapeStr) >= 3 + and _VSCODE_shapeStr[0] == "(" + and _VSCODE_shapeStr[-1] == ")" + and "," in _VSCODE_shapeStr + ): + result["shape"] = _VSCODE_shapeStr + del _VSCODE_shapeStr + _VSCODE_maybeParseTensorShape(var, result) + except TypeError: + pass + + if hasattr(var, "__len__"): + try: + result["count"] = len(var) + except TypeError: + pass + + # return our json object as a string + return _VSCODE_json.dumps(result) diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index 5b6784e4835..414791034bb 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -489,6 +489,13 @@ export namespace Settings { export namespace DataFrameLoading { export const SysPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'dataframes'); + export const HelperPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'dataframes', + 'helpers' + ); export const DataFrameSysImport = `import sys\nsys.path.append("${SysPath.replace(/\\/g, '\\\\')}")`; export const DataFrameInfoImportName = '_VSCODE_InfoImport'; export const DataFrameInfoImport = `import vscodeGetDataFrameInfo as ${DataFrameInfoImportName}`; @@ -501,6 +508,17 @@ export namespace DataFrameLoading { export const VariableInfoFunc = `${VariableInfoImportName}._VSCODE_getVariableInfo`; } +export namespace GetVariableInfo { + export const ScriptPath = path.join( + EXTENSION_ROOT_DIR, + 'pythonFiles', + 'vscode_datascience_helpers', + 'getVariableInfo', + 'vscodeVariableInfo.py' + ); + export const VariableInfoFunc = '_VSCODE_getVariableInfo'; +} + export namespace Identifiers { export const EmptyFileName = '2DB9B899-6519-4E1B-88B0-FA728A274115'; export const GeneratedThemeName = 'ipython-theme'; // This needs to be all lower class and a valid class name. diff --git a/src/client/datascience/jupyter/kernelVariables.ts b/src/client/datascience/jupyter/kernelVariables.ts index 42701590802..83becb08f5a 100644 --- a/src/client/datascience/jupyter/kernelVariables.ts +++ b/src/client/datascience/jupyter/kernelVariables.ts @@ -3,15 +3,17 @@ 'use strict'; import type { nbformat } from '@jupyterlab/coreutils'; import { inject, injectable } from 'inversify'; +import * as path from 'path'; import stripAnsi from 'strip-ansi'; import * as uuid from 'uuid/v4'; import { CancellationToken, Event, EventEmitter, Uri } from 'vscode'; import { PYTHON_LANGUAGE } from '../../common/constants'; import { traceError } from '../../common/logger'; +import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService, IDisposable } from '../../common/types'; import * as localize from '../../common/utils/localize'; -import { DataFrameLoading, Identifiers, Settings } from '../constants'; +import { DataFrameLoading, GetVariableInfo, Identifiers, Settings } from '../constants'; import { ICell, IJupyterVariable, @@ -44,11 +46,15 @@ interface INotebookState { @injectable() export class KernelVariables implements IJupyterVariables { private importedDataFrameScripts = new Map(); + private importedGetVariableInfoScripts = new Map(); private languageToQueryMap = new Map(); private notebookState = new Map(); private refreshEventEmitter = new EventEmitter(); - constructor(@inject(IConfigurationService) private configService: IConfigurationService) {} + constructor( + @inject(IConfigurationService) private configService: IConfigurationService, + @inject(IFileSystem) private fs: IFileSystem + ) {} public get refreshRequired(): Event { return this.refreshEventEmitter.event; @@ -148,6 +154,86 @@ export class KernelVariables implements IJupyterVariables { } private async importDataFrameScripts(notebook: INotebook, token?: CancellationToken): Promise { + const key = notebook.identity.toString(); + if (!this.importedDataFrameScripts.get(key)) { + // Clear our flag if the notebook disposes or restarts + const disposables: IDisposable[] = []; + const handler = () => { + this.importedDataFrameScripts.delete(key); + disposables.forEach((d) => d.dispose()); + }; + disposables.push(notebook.onDisposed(handler)); + disposables.push(notebook.onKernelChanged(handler)); + disposables.push(notebook.onKernelRestarted(handler)); + + // First put the code from our helper files into the notebook + await this.importDataFrameHelperScripts(notebook, token); + + // Add in the actual imports that we need + const fullCode = `${DataFrameLoading.DataFrameInfoImport}\n${DataFrameLoading.DataFrameRowImport}\n${DataFrameLoading.VariableInfoImport}`; + await notebook.execute(fullCode, Identifiers.EmptyFileName, 0, uuid(), token, true); + this.importedDataFrameScripts.set(notebook.identity.toString(), true); + } + } + + private async importGetVariableInfoScripts(notebook: INotebook, token?: CancellationToken): Promise { + const key = notebook.identity.toString(); + if (!this.importedGetVariableInfoScripts.get(key)) { + // Clear our flag if the notebook disposes or restarts + // IANHU: Same as dataframe code. Refactor? + const disposables: IDisposable[] = []; + const handler = () => { + this.importedGetVariableInfoScripts.delete(key); + disposables.forEach((d) => d.dispose()); + }; + disposables.push(notebook.onDisposed(handler)); + disposables.push(notebook.onKernelChanged(handler)); + disposables.push(notebook.onKernelRestarted(handler)); + } + + await this.importGetVariableInfoHelperScripts(notebook, token); + } + + private async importGetVariableInfoHelperScripts(notebook: INotebook, token?: CancellationToken) { + // First pull all our helper files from the helper script directory + //const scriptFile = await this.fs.searchLocal('**/*.py', DataFrameLoading.HelperPath); + + //await Promise.all( + //scriptFiles.map(async (file) => { + //return this.runScriptFile(notebook, path.join(DataFrameLoading.HelperPath, file), token); + //}) + //); + + await this.runScriptFile(notebook, GetVariableInfo.ScriptPath, token); + + // IANHU: Check my sync here? + traceError('just testing'); + } + + private async importDataFrameHelperScripts(notebook: INotebook, token?: CancellationToken) { + // First pull all our helper files from the helper script directory + const scriptFiles = await this.fs.searchLocal('**/*.py', DataFrameLoading.HelperPath); + + await Promise.all( + scriptFiles.map(async (file) => { + return this.runScriptFile(notebook, path.join(DataFrameLoading.HelperPath, file), token); + }) + ); + + // IANHU: Check my sync here? + traceError('just testing'); + } + + private async runScriptFile(notebook: INotebook, scriptFile: string, token?: CancellationToken) { + if (this.fs.localFileExists(scriptFile)) { + const fileContents = await this.fs.readFile(Uri.parse(scriptFile)); + return notebook.execute(fileContents, Identifiers.EmptyFileName, 0, uuid(), token, true); + } else { + traceError('Cannot run non-existant script file'); + } + } + + private async importDataFrameScriptsOld(notebook: INotebook, token?: CancellationToken): Promise { const key = notebook.identity.toString(); if (!this.importedDataFrameScripts.get(key)) { // Clear our flag if the notebook disposes or restarts @@ -172,11 +258,12 @@ export class KernelVariables implements IJupyterVariables { token?: CancellationToken ): Promise { // Import the data frame script directory if we haven't already - await this.importDataFrameScripts(notebook, token); + //await this.importDataFrameScripts(notebook, token); + await this.importGetVariableInfoHelperScripts(notebook, token); // Then execute a call to get the info and turn it into JSON const results = await notebook.execute( - `print(${DataFrameLoading.VariableInfoFunc}(${targetVariable.name}))`, + `print(${GetVariableInfo.VariableInfoFunc}(${targetVariable.name}))`, Identifiers.EmptyFileName, 0, uuid(), From f1230af64bc31c1c0f6fb079a83cebd86df11b3a Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Mon, 30 Nov 2020 14:37:11 -0800 Subject: [PATCH 02/11] dataviewer also working on docker --- .../dataframes/vscodeDataFrame.py | 96 +++++++++++++++++++ src/client/datascience/constants.ts | 10 +- .../datascience/jupyter/kernelVariables.ts | 80 ++++++---------- 3 files changed, 129 insertions(+), 57 deletions(-) create mode 100644 pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrame.py diff --git a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrame.py b/pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrame.py new file mode 100644 index 00000000000..66324b27c11 --- /dev/null +++ b/pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrame.py @@ -0,0 +1,96 @@ +import pandas as _VSCODE_pd +import builtins as _VSCODE_builtins +import json as _VSCODE_json +import pandas.io.json as _VSCODE_pd_json + +# Function that converts the var passed in into a pandas data frame if possible +def _VSCODE_convertToDataFrame(df): + if isinstance(df, list): + df = _VSCODE_pd.DataFrame(df) + elif isinstance(df, _VSCODE_pd.Series): + df = _VSCODE_pd.Series.to_frame(df) + elif isinstance(df, dict): + df = _VSCODE_pd.Series(df) + df = _VSCODE_pd.Series.to_frame(df) + elif hasattr(df, "toPandas"): + df = df.toPandas() + else: + """Disabling bandit warning for try, except, pass. We want to swallow all exceptions here to not crash on + variable fetching""" + try: + temp = _VSCODE_pd.DataFrame(df) + df = temp + except: # nosec + pass + return df + + +# Function to compute row count for a value +def _VSCODE_getRowCount(var): + if hasattr(var, "shape"): + try: + # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it + if isinstance(var.shape, tuple): + return var.shape[0] + except TypeError: + return 0 + elif hasattr(var, "__len__"): + try: + return _VSCODE_builtins.len(var) + except TypeError: + return 0 + +# Function to retrieve a set of rows for a data frame +def _VSCODE_getDataFrameRows(df, start, end): + df = _VSCODE_convertToDataFrame(df) + + # Turn into JSON using pandas. We use pandas because it's about 3 orders of magnitude faster to turn into JSON + rows = df.iloc[start:end] + return _VSCODE_pd_json.to_json(None, rows, orient="table", date_format="iso") + +# Function to get info on the passed in data frame +def _VSCODE_getDataFrameInfo(df): + df = _VSCODE_convertToDataFrame(df) + rowCount = _VSCODE_getRowCount(df) + + # If any rows, use pandas json to convert a single row to json. Extract + # the column names and types from the json so we match what we'll fetch when + # we ask for all of the rows + if rowCount: + try: + row = df.iloc[0:1] + json_row = _VSCODE_pd_json.to_json(None, row, date_format="iso") + columnNames = list(_VSCODE_json.loads(json_row)) + except: + columnNames = list(df) + else: + columnNames = list(df) + + # Compute the index column. It may have been renamed + indexColumn = df.index.name if df.index.name else "index" + columnTypes = _VSCODE_builtins.list(df.dtypes) + + # Make sure the index column exists + if indexColumn not in columnNames: + columnNames.insert(0, indexColumn) + columnTypes.insert(0, "int64") + + # Then loop and generate our output json + columns = [] + for n in _VSCODE_builtins.range(0, _VSCODE_builtins.len(columnNames)): + column_type = columnTypes[n] + column_name = str(columnNames[n]) + colobj = {} + colobj["key"] = column_name + colobj["name"] = column_name + colobj["type"] = str(column_type) + columns.append(colobj) + + # Save this in our target + target = {} + target["columns"] = columns + target["indexColumn"] = indexColumn + target["rowCount"] = rowCount + + # return our json object as a string + return _VSCODE_json.dumps(target) diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index 414791034bb..4cebde330d2 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -489,20 +489,22 @@ export namespace Settings { export namespace DataFrameLoading { export const SysPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'dataframes'); - export const HelperPath = path.join( + export const ScriptPath = path.join( EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'dataframes', - 'helpers' + 'vscodeDataFrame.py' ); export const DataFrameSysImport = `import sys\nsys.path.append("${SysPath.replace(/\\/g, '\\\\')}")`; export const DataFrameInfoImportName = '_VSCODE_InfoImport'; export const DataFrameInfoImport = `import vscodeGetDataFrameInfo as ${DataFrameInfoImportName}`; - export const DataFrameInfoFunc = `${DataFrameInfoImportName}._VSCODE_getDataFrameInfo`; + //export const DataFrameInfoFunc = `${DataFrameInfoImportName}._VSCODE_getDataFrameInfo`; + export const DataFrameInfoFunc = '_VSCODE_getDataFrameInfo'; export const DataFrameRowImportName = '_VSCODE_RowImport'; export const DataFrameRowImport = `import vscodeGetDataFrameRows as ${DataFrameRowImportName}`; - export const DataFrameRowFunc = `${DataFrameRowImportName}._VSCODE_getDataFrameRows`; + //export const DataFrameRowFunc = `${DataFrameRowImportName}._VSCODE_getDataFrameRows`; + export const DataFrameRowFunc = '_VSCODE_getDataFrameRows'; export const VariableInfoImportName = '_VSCODE_VariableImport'; export const VariableInfoImport = `import vscodeGetVariableInfo as ${VariableInfoImportName}`; export const VariableInfoFunc = `${VariableInfoImportName}._VSCODE_getVariableInfo`; diff --git a/src/client/datascience/jupyter/kernelVariables.ts b/src/client/datascience/jupyter/kernelVariables.ts index 83becb08f5a..ed3ceb78b86 100644 --- a/src/client/datascience/jupyter/kernelVariables.ts +++ b/src/client/datascience/jupyter/kernelVariables.ts @@ -167,11 +167,12 @@ export class KernelVariables implements IJupyterVariables { disposables.push(notebook.onKernelRestarted(handler)); // First put the code from our helper files into the notebook - await this.importDataFrameHelperScripts(notebook, token); + //await this.importDataFrameHelperScripts(notebook, token); + await this.runScriptFile(notebook, DataFrameLoading.ScriptPath, token); // Add in the actual imports that we need - const fullCode = `${DataFrameLoading.DataFrameInfoImport}\n${DataFrameLoading.DataFrameRowImport}\n${DataFrameLoading.VariableInfoImport}`; - await notebook.execute(fullCode, Identifiers.EmptyFileName, 0, uuid(), token, true); + //const fullCode = `${DataFrameLoading.DataFrameInfoImport}\n${DataFrameLoading.DataFrameRowImport}\n${DataFrameLoading.VariableInfoImport}`; + //await notebook.execute(fullCode, Identifiers.EmptyFileName, 0, uuid(), token, true); this.importedDataFrameScripts.set(notebook.identity.toString(), true); } } @@ -189,39 +190,12 @@ export class KernelVariables implements IJupyterVariables { disposables.push(notebook.onDisposed(handler)); disposables.push(notebook.onKernelChanged(handler)); disposables.push(notebook.onKernelRestarted(handler)); - } - - await this.importGetVariableInfoHelperScripts(notebook, token); - } - - private async importGetVariableInfoHelperScripts(notebook: INotebook, token?: CancellationToken) { - // First pull all our helper files from the helper script directory - //const scriptFile = await this.fs.searchLocal('**/*.py', DataFrameLoading.HelperPath); - //await Promise.all( - //scriptFiles.map(async (file) => { - //return this.runScriptFile(notebook, path.join(DataFrameLoading.HelperPath, file), token); - //}) - //); - - await this.runScriptFile(notebook, GetVariableInfo.ScriptPath, token); - - // IANHU: Check my sync here? - traceError('just testing'); - } + //await this.importGetVariableInfoHelperScripts(notebook, token); + await this.runScriptFile(notebook, GetVariableInfo.ScriptPath, token); - private async importDataFrameHelperScripts(notebook: INotebook, token?: CancellationToken) { - // First pull all our helper files from the helper script directory - const scriptFiles = await this.fs.searchLocal('**/*.py', DataFrameLoading.HelperPath); - - await Promise.all( - scriptFiles.map(async (file) => { - return this.runScriptFile(notebook, path.join(DataFrameLoading.HelperPath, file), token); - }) - ); - - // IANHU: Check my sync here? - traceError('just testing'); + this.importedGetVariableInfoScripts.set(notebook.identity.toString(), true); + } } private async runScriptFile(notebook: INotebook, scriptFile: string, token?: CancellationToken) { @@ -233,24 +207,24 @@ export class KernelVariables implements IJupyterVariables { } } - private async importDataFrameScriptsOld(notebook: INotebook, token?: CancellationToken): Promise { - const key = notebook.identity.toString(); - if (!this.importedDataFrameScripts.get(key)) { - // Clear our flag if the notebook disposes or restarts - const disposables: IDisposable[] = []; - const handler = () => { - this.importedDataFrameScripts.delete(key); - disposables.forEach((d) => d.dispose()); - }; - disposables.push(notebook.onDisposed(handler)); - disposables.push(notebook.onKernelChanged(handler)); - disposables.push(notebook.onKernelRestarted(handler)); - - const fullCode = `${DataFrameLoading.DataFrameSysImport}\n${DataFrameLoading.DataFrameInfoImport}\n${DataFrameLoading.DataFrameRowImport}\n${DataFrameLoading.VariableInfoImport}`; - await notebook.execute(fullCode, Identifiers.EmptyFileName, 0, uuid(), token, true); - this.importedDataFrameScripts.set(notebook.identity.toString(), true); - } - } + //private async importDataFrameScriptsOld(notebook: INotebook, token?: CancellationToken): Promise { + //const key = notebook.identity.toString(); + //if (!this.importedDataFrameScripts.get(key)) { + //// Clear our flag if the notebook disposes or restarts + //const disposables: IDisposable[] = []; + //const handler = () => { + //this.importedDataFrameScripts.delete(key); + //disposables.forEach((d) => d.dispose()); + //}; + //disposables.push(notebook.onDisposed(handler)); + //disposables.push(notebook.onKernelChanged(handler)); + //disposables.push(notebook.onKernelRestarted(handler)); + + //const fullCode = `${DataFrameLoading.DataFrameSysImport}\n${DataFrameLoading.DataFrameInfoImport}\n${DataFrameLoading.DataFrameRowImport}\n${DataFrameLoading.VariableInfoImport}`; + //await notebook.execute(fullCode, Identifiers.EmptyFileName, 0, uuid(), token, true); + //this.importedDataFrameScripts.set(notebook.identity.toString(), true); + //} + //} private async getFullVariable( targetVariable: IJupyterVariable, @@ -259,7 +233,7 @@ export class KernelVariables implements IJupyterVariables { ): Promise { // Import the data frame script directory if we haven't already //await this.importDataFrameScripts(notebook, token); - await this.importGetVariableInfoHelperScripts(notebook, token); + await this.importGetVariableInfoScripts(notebook, token); // Then execute a call to get the info and turn it into JSON const results = await notebook.execute( From 3c4558a303b10e7f37590303f8c76c259cfb44f6 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 1 Dec 2020 09:58:16 -0800 Subject: [PATCH 03/11] debugger variables also working, ready for cleanup --- .../datascience/jupyter/debuggerVariables.ts | 57 ++++++++++++++----- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/src/client/datascience/jupyter/debuggerVariables.ts b/src/client/datascience/jupyter/debuggerVariables.ts index b5ac51112b6..c455ed083d0 100644 --- a/src/client/datascience/jupyter/debuggerVariables.ts +++ b/src/client/datascience/jupyter/debuggerVariables.ts @@ -3,13 +3,14 @@ 'use strict'; import { inject, injectable, named } from 'inversify'; -import { DebugAdapterTracker, Disposable, Event, EventEmitter } from 'vscode'; +import { DebugAdapterTracker, Disposable, Event, EventEmitter, Uri } from 'vscode'; import { DebugProtocol } from 'vscode-debugprotocol'; import { IDebugService } from '../../common/application/types'; import { traceError } from '../../common/logger'; +import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService, Resource } from '../../common/types'; import { sendTelemetryEvent } from '../../telemetry'; -import { DataFrameLoading, Identifiers, Telemetry } from '../constants'; +import { DataFrameLoading, GetVariableInfo, Identifiers, Telemetry } from '../constants'; import { DebugLocationTracker } from '../debugLocationTracker'; import { IConditionalJupyterVariables, @@ -28,12 +29,14 @@ export class DebuggerVariables extends DebugLocationTracker implements IConditionalJupyterVariables, DebugAdapterTracker { private refreshEventEmitter = new EventEmitter(); private lastKnownVariables: IJupyterVariable[] = []; - private importedIntoKernel = new Set(); + private importedDataFrameScriptsIntoKernel = new Set(); + private importedGetVariableInfoScriptsIntoKernel = new Set(); private watchedNotebooks = new Map(); private debuggingStarted = false; constructor( @inject(IJupyterDebugService) @named(Identifiers.MULTIPLEXING_DEBUGSERVICE) private debugService: IDebugService, - @inject(IConfigurationService) private configService: IConfigurationService + @inject(IConfigurationService) private configService: IConfigurationService, + @inject(IFileSystem) private fs: IFileSystem ) { super(undefined); } @@ -194,7 +197,8 @@ export class DebuggerVariables extends DebugLocationTracker this.refreshEventEmitter.fire(); const key = this.debugService.activeDebugSession?.id; if (key) { - this.importedIntoKernel.delete(key); + this.importedDataFrameScriptsIntoKernel.delete(key); + this.importedGetVariableInfoScriptsIntoKernel.delete(key); } } } @@ -217,7 +221,8 @@ export class DebuggerVariables extends DebugLocationTracker } private resetImport(key: string) { - this.importedIntoKernel.delete(key); + this.importedDataFrameScriptsIntoKernel.delete(key); + this.importedGetVariableInfoScriptsIntoKernel.delete(key); } // tslint:disable-next-line: no-any @@ -243,25 +248,49 @@ export class DebuggerVariables extends DebugLocationTracker try { // Run our dataframe scripts only once per session because they're slow const key = this.debugService.activeDebugSession?.id; - if (key && !this.importedIntoKernel.has(key)) { - await this.evaluate(DataFrameLoading.DataFrameSysImport); - await this.evaluate(DataFrameLoading.DataFrameInfoImport); - await this.evaluate(DataFrameLoading.DataFrameRowImport); - await this.evaluate(DataFrameLoading.VariableInfoImport); - this.importedIntoKernel.add(key); + if (key && !this.importedDataFrameScriptsIntoKernel.has(key)) { + //await this.evaluate(DataFrameLoading.DataFrameSysImport); + //await this.evaluate(DataFrameLoading.DataFrameInfoImport); + //await this.evaluate(DataFrameLoading.DataFrameRowImport); + await this.evaluateScriptFile(DataFrameLoading.ScriptPath); + this.importedDataFrameScriptsIntoKernel.add(key); } } catch (exc) { traceError('Error attempting to import in debugger', exc); } } + private async importGetVariableInfoScripts(): Promise { + try { + // Run our variable info scripts only once per session because they're slow + const key = this.debugService.activeDebugSession?.id; + if (key && !this.importedGetVariableInfoScriptsIntoKernel.has(key)) { + //await this.evaluate(DataFrameLoading.VariableInfoImport); + await this.evaluateScriptFile(GetVariableInfo.ScriptPath); + this.importedGetVariableInfoScriptsIntoKernel.add(key); + } + } catch (exc) { + traceError('Error attempting to import in debugger', exc); + } + } + + private async evaluateScriptFile(fileName: string): Promise { + if (this.fs.localFileExists(fileName)) { + const fileContents = await this.fs.readFile(Uri.parse(fileName)); + return this.evaluate(fileContents); + } else { + traceError('Cannot run non-existant script file'); + } + } + private async getFullVariable(variable: IJupyterVariable): Promise { // See if we imported or not into the kernel our special function - await this.importDataFrameScripts(); + //await this.importDataFrameScripts(); + await this.importGetVariableInfoScripts(); // Then eval calling the variable info function with our target variable const results = await this.evaluate( - `${DataFrameLoading.VariableInfoFunc}(${variable.name})`, + `${GetVariableInfo.VariableInfoFunc}(${variable.name})`, // tslint:disable-next-line: no-any (variable as any).frameId ); From aef1f0bbf898c3252ff48888f1adfc322c0633a6 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 1 Dec 2020 10:01:15 -0800 Subject: [PATCH 04/11] await on local file exists --- src/client/datascience/jupyter/debuggerVariables.ts | 2 +- src/client/datascience/jupyter/kernelVariables.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/datascience/jupyter/debuggerVariables.ts b/src/client/datascience/jupyter/debuggerVariables.ts index c455ed083d0..5b2eafc1910 100644 --- a/src/client/datascience/jupyter/debuggerVariables.ts +++ b/src/client/datascience/jupyter/debuggerVariables.ts @@ -275,7 +275,7 @@ export class DebuggerVariables extends DebugLocationTracker } private async evaluateScriptFile(fileName: string): Promise { - if (this.fs.localFileExists(fileName)) { + if (await this.fs.localFileExists(fileName)) { const fileContents = await this.fs.readFile(Uri.parse(fileName)); return this.evaluate(fileContents); } else { diff --git a/src/client/datascience/jupyter/kernelVariables.ts b/src/client/datascience/jupyter/kernelVariables.ts index ed3ceb78b86..38dc98d3a46 100644 --- a/src/client/datascience/jupyter/kernelVariables.ts +++ b/src/client/datascience/jupyter/kernelVariables.ts @@ -199,7 +199,7 @@ export class KernelVariables implements IJupyterVariables { } private async runScriptFile(notebook: INotebook, scriptFile: string, token?: CancellationToken) { - if (this.fs.localFileExists(scriptFile)) { + if (await this.fs.localFileExists(scriptFile)) { const fileContents = await this.fs.readFile(Uri.parse(scriptFile)); return notebook.execute(fileContents, Identifiers.EmptyFileName, 0, uuid(), token, true); } else { From 6ee5562c663b915e1fc6fc180f0327d59a9b3b9c Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 1 Dec 2020 10:14:18 -0800 Subject: [PATCH 05/11] trim list of constants to just what I need --- src/client/datascience/constants.ts | 11 ----------- src/client/datascience/jupyter/debuggerVariables.ts | 6 +----- src/client/datascience/jupyter/kernelVariables.ts | 1 - 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index 4cebde330d2..43b7878e71c 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -488,7 +488,6 @@ export namespace Settings { } export namespace DataFrameLoading { - export const SysPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'dataframes'); export const ScriptPath = path.join( EXTENSION_ROOT_DIR, 'pythonFiles', @@ -496,18 +495,8 @@ export namespace DataFrameLoading { 'dataframes', 'vscodeDataFrame.py' ); - export const DataFrameSysImport = `import sys\nsys.path.append("${SysPath.replace(/\\/g, '\\\\')}")`; - export const DataFrameInfoImportName = '_VSCODE_InfoImport'; - export const DataFrameInfoImport = `import vscodeGetDataFrameInfo as ${DataFrameInfoImportName}`; - //export const DataFrameInfoFunc = `${DataFrameInfoImportName}._VSCODE_getDataFrameInfo`; export const DataFrameInfoFunc = '_VSCODE_getDataFrameInfo'; - export const DataFrameRowImportName = '_VSCODE_RowImport'; - export const DataFrameRowImport = `import vscodeGetDataFrameRows as ${DataFrameRowImportName}`; - //export const DataFrameRowFunc = `${DataFrameRowImportName}._VSCODE_getDataFrameRows`; export const DataFrameRowFunc = '_VSCODE_getDataFrameRows'; - export const VariableInfoImportName = '_VSCODE_VariableImport'; - export const VariableInfoImport = `import vscodeGetVariableInfo as ${VariableInfoImportName}`; - export const VariableInfoFunc = `${VariableInfoImportName}._VSCODE_getVariableInfo`; } export namespace GetVariableInfo { diff --git a/src/client/datascience/jupyter/debuggerVariables.ts b/src/client/datascience/jupyter/debuggerVariables.ts index 5b2eafc1910..d2432df8f34 100644 --- a/src/client/datascience/jupyter/debuggerVariables.ts +++ b/src/client/datascience/jupyter/debuggerVariables.ts @@ -249,9 +249,6 @@ export class DebuggerVariables extends DebugLocationTracker // Run our dataframe scripts only once per session because they're slow const key = this.debugService.activeDebugSession?.id; if (key && !this.importedDataFrameScriptsIntoKernel.has(key)) { - //await this.evaluate(DataFrameLoading.DataFrameSysImport); - //await this.evaluate(DataFrameLoading.DataFrameInfoImport); - //await this.evaluate(DataFrameLoading.DataFrameRowImport); await this.evaluateScriptFile(DataFrameLoading.ScriptPath); this.importedDataFrameScriptsIntoKernel.add(key); } @@ -265,7 +262,6 @@ export class DebuggerVariables extends DebugLocationTracker // Run our variable info scripts only once per session because they're slow const key = this.debugService.activeDebugSession?.id; if (key && !this.importedGetVariableInfoScriptsIntoKernel.has(key)) { - //await this.evaluate(DataFrameLoading.VariableInfoImport); await this.evaluateScriptFile(GetVariableInfo.ScriptPath); this.importedGetVariableInfoScriptsIntoKernel.add(key); } @@ -274,6 +270,7 @@ export class DebuggerVariables extends DebugLocationTracker } } + // Load the given python script file and evaluate the contents private async evaluateScriptFile(fileName: string): Promise { if (await this.fs.localFileExists(fileName)) { const fileContents = await this.fs.readFile(Uri.parse(fileName)); @@ -285,7 +282,6 @@ export class DebuggerVariables extends DebugLocationTracker private async getFullVariable(variable: IJupyterVariable): Promise { // See if we imported or not into the kernel our special function - //await this.importDataFrameScripts(); await this.importGetVariableInfoScripts(); // Then eval calling the variable info function with our target variable diff --git a/src/client/datascience/jupyter/kernelVariables.ts b/src/client/datascience/jupyter/kernelVariables.ts index 38dc98d3a46..e6cf74e0b29 100644 --- a/src/client/datascience/jupyter/kernelVariables.ts +++ b/src/client/datascience/jupyter/kernelVariables.ts @@ -3,7 +3,6 @@ 'use strict'; import type { nbformat } from '@jupyterlab/coreutils'; import { inject, injectable } from 'inversify'; -import * as path from 'path'; import stripAnsi from 'strip-ansi'; import * as uuid from 'uuid/v4'; From 3cd15f894a687a4e21ff6118a76a4ab8775a1f56 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 1 Dec 2020 10:27:21 -0800 Subject: [PATCH 06/11] cut out some unneeded code --- src/client/datascience/jupyter/kernelVariables.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/client/datascience/jupyter/kernelVariables.ts b/src/client/datascience/jupyter/kernelVariables.ts index e6cf74e0b29..4a5a8b804ed 100644 --- a/src/client/datascience/jupyter/kernelVariables.ts +++ b/src/client/datascience/jupyter/kernelVariables.ts @@ -166,12 +166,8 @@ export class KernelVariables implements IJupyterVariables { disposables.push(notebook.onKernelRestarted(handler)); // First put the code from our helper files into the notebook - //await this.importDataFrameHelperScripts(notebook, token); await this.runScriptFile(notebook, DataFrameLoading.ScriptPath, token); - // Add in the actual imports that we need - //const fullCode = `${DataFrameLoading.DataFrameInfoImport}\n${DataFrameLoading.DataFrameRowImport}\n${DataFrameLoading.VariableInfoImport}`; - //await notebook.execute(fullCode, Identifiers.EmptyFileName, 0, uuid(), token, true); this.importedDataFrameScripts.set(notebook.identity.toString(), true); } } @@ -180,7 +176,6 @@ export class KernelVariables implements IJupyterVariables { const key = notebook.identity.toString(); if (!this.importedGetVariableInfoScripts.get(key)) { // Clear our flag if the notebook disposes or restarts - // IANHU: Same as dataframe code. Refactor? const disposables: IDisposable[] = []; const handler = () => { this.importedGetVariableInfoScripts.delete(key); @@ -190,7 +185,6 @@ export class KernelVariables implements IJupyterVariables { disposables.push(notebook.onKernelChanged(handler)); disposables.push(notebook.onKernelRestarted(handler)); - //await this.importGetVariableInfoHelperScripts(notebook, token); await this.runScriptFile(notebook, GetVariableInfo.ScriptPath, token); this.importedGetVariableInfoScripts.set(notebook.identity.toString(), true); From 0b1b1b6cb5ee00a0091f96f5405bdf3726083782 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 1 Dec 2020 10:34:22 -0800 Subject: [PATCH 07/11] add news and cut a few comments out --- news/2 Fixes/4006.md | 1 + .../datascience/jupyter/kernelVariables.ts | 23 ++----------------- 2 files changed, 3 insertions(+), 21 deletions(-) create mode 100644 news/2 Fixes/4006.md diff --git a/news/2 Fixes/4006.md b/news/2 Fixes/4006.md new file mode 100644 index 00000000000..d3da562d12f --- /dev/null +++ b/news/2 Fixes/4006.md @@ -0,0 +1 @@ +Fix variable fetching on remote machines that don't have our scripts files on them. \ No newline at end of file diff --git a/src/client/datascience/jupyter/kernelVariables.ts b/src/client/datascience/jupyter/kernelVariables.ts index 4a5a8b804ed..afa3bc42a84 100644 --- a/src/client/datascience/jupyter/kernelVariables.ts +++ b/src/client/datascience/jupyter/kernelVariables.ts @@ -191,6 +191,7 @@ export class KernelVariables implements IJupyterVariables { } } + // Read in a .py file and execute it silently in the given notebook private async runScriptFile(notebook: INotebook, scriptFile: string, token?: CancellationToken) { if (await this.fs.localFileExists(scriptFile)) { const fileContents = await this.fs.readFile(Uri.parse(scriptFile)); @@ -200,32 +201,12 @@ export class KernelVariables implements IJupyterVariables { } } - //private async importDataFrameScriptsOld(notebook: INotebook, token?: CancellationToken): Promise { - //const key = notebook.identity.toString(); - //if (!this.importedDataFrameScripts.get(key)) { - //// Clear our flag if the notebook disposes or restarts - //const disposables: IDisposable[] = []; - //const handler = () => { - //this.importedDataFrameScripts.delete(key); - //disposables.forEach((d) => d.dispose()); - //}; - //disposables.push(notebook.onDisposed(handler)); - //disposables.push(notebook.onKernelChanged(handler)); - //disposables.push(notebook.onKernelRestarted(handler)); - - //const fullCode = `${DataFrameLoading.DataFrameSysImport}\n${DataFrameLoading.DataFrameInfoImport}\n${DataFrameLoading.DataFrameRowImport}\n${DataFrameLoading.VariableInfoImport}`; - //await notebook.execute(fullCode, Identifiers.EmptyFileName, 0, uuid(), token, true); - //this.importedDataFrameScripts.set(notebook.identity.toString(), true); - //} - //} - private async getFullVariable( targetVariable: IJupyterVariable, notebook: INotebook, token?: CancellationToken ): Promise { - // Import the data frame script directory if we haven't already - //await this.importDataFrameScripts(notebook, token); + // Add in our get variable info script await this.importGetVariableInfoScripts(notebook, token); // Then execute a call to get the info and turn it into JSON From cbda46a67353c781d131f25142e5268d3cc03cbd Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 1 Dec 2020 10:37:55 -0800 Subject: [PATCH 08/11] delete old helper files --- .../dataframes/vscodeDataFrameHelpers.py | 39 ------------- .../dataframes/vscodeGetDataFrameInfo.py | 53 ------------------ .../dataframes/vscodeGetDataFrameRows.py | 11 ---- .../dataframes/vscodeGetVariableInfo.py | 55 ------------------- 4 files changed, 158 deletions(-) delete mode 100644 pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrameHelpers.py delete mode 100644 pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameInfo.py delete mode 100644 pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameRows.py delete mode 100644 pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetVariableInfo.py diff --git a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrameHelpers.py b/pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrameHelpers.py deleted file mode 100644 index d72dbacfc62..00000000000 --- a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrameHelpers.py +++ /dev/null @@ -1,39 +0,0 @@ -import pandas as _VSCODE_pd -import builtins as _VSCODE_builtins - -# Function that converts the var passed in into a pandas data frame if possible -def _VSCODE_convertToDataFrame(df): - if isinstance(df, list): - df = _VSCODE_pd.DataFrame(df) - elif isinstance(df, _VSCODE_pd.Series): - df = _VSCODE_pd.Series.to_frame(df) - elif isinstance(df, dict): - df = _VSCODE_pd.Series(df) - df = _VSCODE_pd.Series.to_frame(df) - elif hasattr(df, "toPandas"): - df = df.toPandas() - else: - """Disabling bandit warning for try, except, pass. We want to swallow all exceptions here to not crash on - variable fetching""" - try: - temp = _VSCODE_pd.DataFrame(df) - df = temp - except: # nosec - pass - return df - - -# Function to compute row count for a value -def _VSCODE_getRowCount(var): - if hasattr(var, "shape"): - try: - # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it - if isinstance(var.shape, tuple): - return var.shape[0] - except TypeError: - return 0 - elif hasattr(var, "__len__"): - try: - return _VSCODE_builtins.len(var) - except TypeError: - return 0 diff --git a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameInfo.py b/pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameInfo.py deleted file mode 100644 index 753a447dc95..00000000000 --- a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameInfo.py +++ /dev/null @@ -1,53 +0,0 @@ -# Query Jupyter server for the info about a dataframe -import json as _VSCODE_json -import pandas as _VSCODE_pd -import pandas.io.json as _VSCODE_pd_json -import builtins as _VSCODE_builtins -import vscodeDataFrameHelpers as _VSCODE_dataFrameHelpers - -# Function to do our work. It will return the object -def _VSCODE_getDataFrameInfo(df): - df = _VSCODE_dataFrameHelpers._VSCODE_convertToDataFrame(df) - rowCount = _VSCODE_dataFrameHelpers._VSCODE_getRowCount(df) - - # If any rows, use pandas json to convert a single row to json. Extract - # the column names and types from the json so we match what we'll fetch when - # we ask for all of the rows - if rowCount: - try: - row = df.iloc[0:1] - json_row = _VSCODE_pd_json.to_json(None, row, date_format="iso") - columnNames = list(_VSCODE_json.loads(json_row)) - except: - columnNames = list(df) - else: - columnNames = list(df) - - # Compute the index column. It may have been renamed - indexColumn = df.index.name if df.index.name else "index" - columnTypes = _VSCODE_builtins.list(df.dtypes) - - # Make sure the index column exists - if indexColumn not in columnNames: - columnNames.insert(0, indexColumn) - columnTypes.insert(0, "int64") - - # Then loop and generate our output json - columns = [] - for n in _VSCODE_builtins.range(0, _VSCODE_builtins.len(columnNames)): - column_type = columnTypes[n] - column_name = str(columnNames[n]) - colobj = {} - colobj["key"] = column_name - colobj["name"] = column_name - colobj["type"] = str(column_type) - columns.append(colobj) - - # Save this in our target - target = {} - target["columns"] = columns - target["indexColumn"] = indexColumn - target["rowCount"] = rowCount - - # return our json object as a string - return _VSCODE_json.dumps(target) diff --git a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameRows.py b/pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameRows.py deleted file mode 100644 index bf176de337e..00000000000 --- a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetDataFrameRows.py +++ /dev/null @@ -1,11 +0,0 @@ -# Query for the rows of a data frame -import pandas.io.json as _VSCODE_pd_json -import vscodeDataFrameHelpers as _VSCODE_dataFrameHelpers - -# Function to retrieve a set of rows for a data frame -def _VSCODE_getDataFrameRows(df, start, end): - df = _VSCODE_dataFrameHelpers._VSCODE_convertToDataFrame(df) - - # Turn into JSON using pandas. We use pandas because it's about 3 orders of magnitude faster to turn into JSON - rows = df.iloc[start:end] - return _VSCODE_pd_json.to_json(None, rows, orient="table", date_format="iso") diff --git a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetVariableInfo.py b/pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetVariableInfo.py deleted file mode 100644 index 6b94418a291..00000000000 --- a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetVariableInfo.py +++ /dev/null @@ -1,55 +0,0 @@ -# Query Jupyter server for the info about a dataframe -import json as _VSCODE_json -import pandas as _VSCODE_pd -import pandas.io.json as _VSCODE_pd_json -import builtins as _VSCODE_builtins -import vscodeDataFrameHelpers as _VSCODE_dataFrameHelpers - - -def _VSCODE_maybeParseTensorShape(var, result): - try: - vartype = type(var) - if (hasattr(vartype, "__name__")) and vartype.__name__ == "Tensor": - varshape = str(var.shape) - start = varshape.index("[") - end = varshape.index("]") - if start > 0 and end > 0: - res = "(" + varshape[start + 1 : end] + ")" - result["shape"] = res - except TypeError: - pass - - -# Function to do our work. It will return the object -def _VSCODE_getVariableInfo(var): - # Start out without the information - result = {} - result["shape"] = "" - result["count"] = 0 - - # Find shape and count if available - if hasattr(var, "shape"): - try: - # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it - if isinstance(var.shape, tuple): - _VSCODE_shapeStr = str(var.shape) - if ( - len(_VSCODE_shapeStr) >= 3 - and _VSCODE_shapeStr[0] == "(" - and _VSCODE_shapeStr[-1] == ")" - and "," in _VSCODE_shapeStr - ): - result["shape"] = _VSCODE_shapeStr - del _VSCODE_shapeStr - _VSCODE_maybeParseTensorShape(var, result) - except TypeError: - pass - - if hasattr(var, "__len__"): - try: - result["count"] = len(var) - except TypeError: - pass - - # return our json object as a string - return _VSCODE_json.dumps(result) From 2616fe0d33f5a6da55aca2f68359d7c56bc5fe84 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 1 Dec 2020 10:43:13 -0800 Subject: [PATCH 09/11] put old file name back --- .../{vscodeVariableInfo.py => vscodeGetVariableInfo.py} | 0 src/client/datascience/constants.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename pythonFiles/vscode_datascience_helpers/getVariableInfo/{vscodeVariableInfo.py => vscodeGetVariableInfo.py} (100%) diff --git a/pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeVariableInfo.py b/pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeGetVariableInfo.py similarity index 100% rename from pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeVariableInfo.py rename to pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeGetVariableInfo.py diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index 43b7878e71c..5f5115569cd 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -505,7 +505,7 @@ export namespace GetVariableInfo { 'pythonFiles', 'vscode_datascience_helpers', 'getVariableInfo', - 'vscodeVariableInfo.py' + 'vscodeGetVariableInfo.py' ); export const VariableInfoFunc = '_VSCODE_getVariableInfo'; } From c8c2f454879e45fe5534739c67e9a24bd1d41505 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 1 Dec 2020 13:38:07 -0800 Subject: [PATCH 10/11] revert debugger back to using imports --- src/client/datascience/constants.ts | 29 ++++++++++++------- .../datascience/jupyter/debuggerVariables.ts | 22 +++++--------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index 5f5115569cd..784ab8d0717 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -488,26 +488,35 @@ export namespace Settings { } export namespace DataFrameLoading { - export const ScriptPath = path.join( - EXTENSION_ROOT_DIR, - 'pythonFiles', - 'vscode_datascience_helpers', - 'dataframes', - 'vscodeDataFrame.py' - ); + export const SysPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'dataframes'); + export const DataFrameSysImport = `import sys\nsys.path.append("${SysPath.replace(/\\/g, '\\\\')}")`; + export const ScriptPath = path.join(SysPath, 'vscodeDataFrame.py'); + export const DataFrameInfoFunc = '_VSCODE_getDataFrameInfo'; export const DataFrameRowFunc = '_VSCODE_getDataFrameRows'; + + // Constants for the debugger which imports the script files + export const DataFrameImportName = '_VSCODE_DataFrameImport'; + export const DataFrameImport = `import vscodeDataFrame as ${DataFrameImportName}`; + export const DataFrameInfoImportFunc = `${DataFrameImportName}._VSCODE_getDataFrameInfo`; + export const DataFrameRowImportFunc = `${DataFrameImportName}._VSCODE_getDataFrameRows`; } export namespace GetVariableInfo { - export const ScriptPath = path.join( + export const SysPath = path.join( EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', - 'getVariableInfo', - 'vscodeGetVariableInfo.py' + 'getVariableInfo' ); + export const GetVariableInfoSysImport = `import sys\nsys.path.append("${SysPath.replace(/\\/g, '\\\\')}")`; + export const ScriptPath = path.join(SysPath, 'vscodeGetVariableInfo.py'); export const VariableInfoFunc = '_VSCODE_getVariableInfo'; + + // Constants for the debugger which imports the script files + export const VariableInfoImportName = '_VSCODE_VariableImport'; + export const VariableInfoImport = `import vscodeGetVariableInfo as ${VariableInfoImportName}`; + export const VariableInfoImportFunc = `${VariableInfoImportName}._VSCODE_getVariableInfo`; } export namespace Identifiers { diff --git a/src/client/datascience/jupyter/debuggerVariables.ts b/src/client/datascience/jupyter/debuggerVariables.ts index d2432df8f34..93d52ad5961 100644 --- a/src/client/datascience/jupyter/debuggerVariables.ts +++ b/src/client/datascience/jupyter/debuggerVariables.ts @@ -112,7 +112,7 @@ export class DebuggerVariables extends DebugLocationTracker // Then eval calling the main function with our target variable const results = await this.evaluate( - `${DataFrameLoading.DataFrameInfoFunc}(${targetVariable.name})`, + `${DataFrameLoading.DataFrameInfoImportFunc}(${targetVariable.name})`, // tslint:disable-next-line: no-any (targetVariable as any).frameId ); @@ -157,7 +157,7 @@ export class DebuggerVariables extends DebugLocationTracker for (let pos = start; pos < end; pos += chunkSize) { const chunkEnd = Math.min(pos + chunkSize, minnedEnd); const results = await this.evaluate( - `${DataFrameLoading.DataFrameRowFunc}(${targetVariable.name}, ${pos}, ${chunkEnd})`, + `${DataFrameLoading.DataFrameRowImportFunc}(${targetVariable.name}, ${pos}, ${chunkEnd})`, // tslint:disable-next-line: no-any (targetVariable as any).frameId ); @@ -249,7 +249,8 @@ export class DebuggerVariables extends DebugLocationTracker // Run our dataframe scripts only once per session because they're slow const key = this.debugService.activeDebugSession?.id; if (key && !this.importedDataFrameScriptsIntoKernel.has(key)) { - await this.evaluateScriptFile(DataFrameLoading.ScriptPath); + await this.evaluate(DataFrameLoading.DataFrameSysImport); + await this.evaluate(DataFrameLoading.DataFrameImport); this.importedDataFrameScriptsIntoKernel.add(key); } } catch (exc) { @@ -262,7 +263,8 @@ export class DebuggerVariables extends DebugLocationTracker // Run our variable info scripts only once per session because they're slow const key = this.debugService.activeDebugSession?.id; if (key && !this.importedGetVariableInfoScriptsIntoKernel.has(key)) { - await this.evaluateScriptFile(GetVariableInfo.ScriptPath); + await this.evaluate(GetVariableInfo.GetVariableInfoSysImport); + await this.evaluate(GetVariableInfo.VariableInfoImport); this.importedGetVariableInfoScriptsIntoKernel.add(key); } } catch (exc) { @@ -270,23 +272,13 @@ export class DebuggerVariables extends DebugLocationTracker } } - // Load the given python script file and evaluate the contents - private async evaluateScriptFile(fileName: string): Promise { - if (await this.fs.localFileExists(fileName)) { - const fileContents = await this.fs.readFile(Uri.parse(fileName)); - return this.evaluate(fileContents); - } else { - traceError('Cannot run non-existant script file'); - } - } - private async getFullVariable(variable: IJupyterVariable): Promise { // See if we imported or not into the kernel our special function await this.importGetVariableInfoScripts(); // Then eval calling the variable info function with our target variable const results = await this.evaluate( - `${GetVariableInfo.VariableInfoFunc}(${variable.name})`, + `${GetVariableInfo.VariableInfoImportFunc}(${variable.name})`, // tslint:disable-next-line: no-any (variable as any).frameId ); From a34e1bce6b0f375685146fd44b79c952409530ed Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 1 Dec 2020 15:23:33 -0800 Subject: [PATCH 11/11] lint and black format check --- .../dataframes/vscodeDataFrame.py | 2 ++ .../getVariableInfo/vscodeGetVariableInfo.py | 1 + src/client/datascience/jupyter/debuggerVariables.ts | 6 ++---- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrame.py b/pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrame.py index 66324b27c11..504f3d4f215 100644 --- a/pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrame.py +++ b/pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrame.py @@ -40,6 +40,7 @@ def _VSCODE_getRowCount(var): except TypeError: return 0 + # Function to retrieve a set of rows for a data frame def _VSCODE_getDataFrameRows(df, start, end): df = _VSCODE_convertToDataFrame(df) @@ -48,6 +49,7 @@ def _VSCODE_getDataFrameRows(df, start, end): rows = df.iloc[start:end] return _VSCODE_pd_json.to_json(None, rows, orient="table", date_format="iso") + # Function to get info on the passed in data frame def _VSCODE_getDataFrameInfo(df): df = _VSCODE_convertToDataFrame(df) diff --git a/pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeGetVariableInfo.py b/pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeGetVariableInfo.py index 7d29bbe4728..74ba4e3ff52 100644 --- a/pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeGetVariableInfo.py +++ b/pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeGetVariableInfo.py @@ -2,6 +2,7 @@ import json as _VSCODE_json import builtins as _VSCODE_builtins + def _VSCODE_maybeParseTensorShape(var, result): try: vartype = type(var) diff --git a/src/client/datascience/jupyter/debuggerVariables.ts b/src/client/datascience/jupyter/debuggerVariables.ts index 93d52ad5961..dd791ef83e5 100644 --- a/src/client/datascience/jupyter/debuggerVariables.ts +++ b/src/client/datascience/jupyter/debuggerVariables.ts @@ -3,11 +3,10 @@ 'use strict'; import { inject, injectable, named } from 'inversify'; -import { DebugAdapterTracker, Disposable, Event, EventEmitter, Uri } from 'vscode'; +import { DebugAdapterTracker, Disposable, Event, EventEmitter } from 'vscode'; import { DebugProtocol } from 'vscode-debugprotocol'; import { IDebugService } from '../../common/application/types'; import { traceError } from '../../common/logger'; -import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService, Resource } from '../../common/types'; import { sendTelemetryEvent } from '../../telemetry'; import { DataFrameLoading, GetVariableInfo, Identifiers, Telemetry } from '../constants'; @@ -35,8 +34,7 @@ export class DebuggerVariables extends DebugLocationTracker private debuggingStarted = false; constructor( @inject(IJupyterDebugService) @named(Identifiers.MULTIPLEXING_DEBUGSERVICE) private debugService: IDebugService, - @inject(IConfigurationService) private configService: IConfigurationService, - @inject(IFileSystem) private fs: IFileSystem + @inject(IConfigurationService) private configService: IConfigurationService ) { super(undefined); }