Fix variable fetching on remote machines

news/2 Fixes/4006.md

@@ -0,0 +1 @@
+Fix variable fetching on remote machines that don't have our scripts files on them.

pythonFiles/vscode_datascience_helpers/dataframes/vscodeDataFrame.py

@@ -0,0 +1,98 @@
+import pandas as _VSCODE_pd
+import builtins as _VSCODE_builtins
+import json as _VSCODE_json
+import pandas.io.json as _VSCODE_pd_json
+
+# Function that converts the var passed in into a pandas data frame if possible
+def _VSCODE_convertToDataFrame(df):
+    if isinstance(df, list):
+        df = _VSCODE_pd.DataFrame(df)
+    elif isinstance(df, _VSCODE_pd.Series):
+        df = _VSCODE_pd.Series.to_frame(df)
+    elif isinstance(df, dict):
+        df = _VSCODE_pd.Series(df)
+        df = _VSCODE_pd.Series.to_frame(df)
+    elif hasattr(df, "toPandas"):
+        df = df.toPandas()
+    else:
+        """Disabling bandit warning for try, except, pass. We want to swallow all exceptions here to not crash on
+        variable fetching"""
+        try:
+            temp = _VSCODE_pd.DataFrame(df)
+            df = temp
+        except:  # nosec
+            pass
+    return df
+
+
+# Function to compute row count for a value
+def _VSCODE_getRowCount(var):
+    if hasattr(var, "shape"):
+        try:
+            # Get a bit more restrictive with exactly what we want to count as a shape, since anything can define it
+            if isinstance(var.shape, tuple):
+                return var.shape[0]
+        except TypeError:
+            return 0
+    elif hasattr(var, "__len__"):
+        try:
+            return _VSCODE_builtins.len(var)
+        except TypeError:
+            return 0
+
+
+# Function to retrieve a set of rows for a data frame
+def _VSCODE_getDataFrameRows(df, start, end):
+    df = _VSCODE_convertToDataFrame(df)
+
+    # Turn into JSON using pandas. We use pandas because it's about 3 orders of magnitude faster to turn into JSON
+    rows = df.iloc[start:end]
+    return _VSCODE_pd_json.to_json(None, rows, orient="table", date_format="iso")
+
+
+# Function to get info on the passed in data frame
+def _VSCODE_getDataFrameInfo(df):
+    df = _VSCODE_convertToDataFrame(df)
+    rowCount = _VSCODE_getRowCount(df)
+
+    # If any rows, use pandas json to convert a single row to json. Extract
+    # the column names and types from the json so we match what we'll fetch when
+    # we ask for all of the rows
+    if rowCount:
+        try:
+            row = df.iloc[0:1]
+            json_row = _VSCODE_pd_json.to_json(None, row, date_format="iso")
+            columnNames = list(_VSCODE_json.loads(json_row))
+        except:
+            columnNames = list(df)
+    else:
+        columnNames = list(df)
+
+    # Compute the index column. It may have been renamed
+    indexColumn = df.index.name if df.index.name else "index"
+    columnTypes = _VSCODE_builtins.list(df.dtypes)
+
+    # Make sure the index column exists
+    if indexColumn not in columnNames:
+        columnNames.insert(0, indexColumn)
+        columnTypes.insert(0, "int64")
+
+    # Then loop and generate our output json
+    columns = []
+    for n in _VSCODE_builtins.range(0, _VSCODE_builtins.len(columnNames)):
+        column_type = columnTypes[n]
+        column_name = str(columnNames[n])
+        colobj = {}
+        colobj["key"] = column_name
+        colobj["name"] = column_name
+        colobj["type"] = str(column_type)
+        columns.append(colobj)
+
+    # Save this in our target
+    target = {}
+    target["columns"] = columns
+    target["indexColumn"] = indexColumn
+    target["rowCount"] = rowCount
+
+    # return our json object as a string
+    return _VSCODE_json.dumps(target)

pythonFiles/vscode_datascience_helpers/dataframes/vscodeGetVariableInfo.py → pythonFiles/vscode_datascience_helpers/getVariableInfo/vscodeGetVariableInfo.py

@@ -1,9 +1,6 @@
 # Query Jupyter server for the info about a dataframe
 import json as _VSCODE_json
-import pandas as _VSCODE_pd
-import pandas.io.json as _VSCODE_pd_json
 import builtins as _VSCODE_builtins
-import vscodeDataFrameHelpers as _VSCODE_dataFrameHelpers
 
 
 def _VSCODE_maybeParseTensorShape(var, result):

src/client/datascience/constants.ts

@@ -490,15 +490,33 @@ export namespace Settings {
 export namespace DataFrameLoading {
     export const SysPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'vscode_datascience_helpers', 'dataframes');
     export const DataFrameSysImport = `import sys\nsys.path.append("${SysPath.replace(/\\/g, '\\\\')}")`;
-    export const DataFrameInfoImportName = '_VSCODE_InfoImport';
-    export const DataFrameInfoImport = `import vscodeGetDataFrameInfo as ${DataFrameInfoImportName}`;
-    export const DataFrameInfoFunc = `${DataFrameInfoImportName}._VSCODE_getDataFrameInfo`;
-    export const DataFrameRowImportName = '_VSCODE_RowImport';
-    export const DataFrameRowImport = `import vscodeGetDataFrameRows as ${DataFrameRowImportName}`;
-    export const DataFrameRowFunc = `${DataFrameRowImportName}._VSCODE_getDataFrameRows`;
+    export const ScriptPath = path.join(SysPath, 'vscodeDataFrame.py');
+
+    export const DataFrameInfoFunc = '_VSCODE_getDataFrameInfo';
+    export const DataFrameRowFunc = '_VSCODE_getDataFrameRows';
+
+    // Constants for the debugger which imports the script files
+    export const DataFrameImportName = '_VSCODE_DataFrameImport';
+    export const DataFrameImport = `import vscodeDataFrame as ${DataFrameImportName}`;
+    export const DataFrameInfoImportFunc = `${DataFrameImportName}._VSCODE_getDataFrameInfo`;
+    export const DataFrameRowImportFunc = `${DataFrameImportName}._VSCODE_getDataFrameRows`;
+}
+
+export namespace GetVariableInfo {
+    export const SysPath = path.join(
+        EXTENSION_ROOT_DIR,
+        'pythonFiles',
+        'vscode_datascience_helpers',
+        'getVariableInfo'
+    );
+    export const GetVariableInfoSysImport = `import sys\nsys.path.append("${SysPath.replace(/\\/g, '\\\\')}")`;
+    export const ScriptPath = path.join(SysPath, 'vscodeGetVariableInfo.py');
+    export const VariableInfoFunc = '_VSCODE_getVariableInfo';
+
+    // Constants for the debugger which imports the script files
     export const VariableInfoImportName = '_VSCODE_VariableImport';
     export const VariableInfoImport = `import vscodeGetVariableInfo as ${VariableInfoImportName}`;
-    export const VariableInfoFunc = `${VariableInfoImportName}._VSCODE_getVariableInfo`;
+    export const VariableInfoImportFunc = `${VariableInfoImportName}._VSCODE_getVariableInfo`;
 }
 
 export namespace Identifiers {

src/client/datascience/jupyter/debuggerVariables.ts

@@ -9,7 +9,7 @@ import { IDebugService } from '../../common/application/types';
 import { traceError } from '../../common/logger';
 import { IConfigurationService, Resource } from '../../common/types';
 import { sendTelemetryEvent } from '../../telemetry';
-import { DataFrameLoading, Identifiers, Telemetry } from '../constants';
+import { DataFrameLoading, GetVariableInfo, Identifiers, Telemetry } from '../constants';
 import { DebugLocationTracker } from '../debugLocationTracker';
 import {
     IConditionalJupyterVariables,
@@ -28,7 +28,8 @@ export class DebuggerVariables extends DebugLocationTracker
     implements IConditionalJupyterVariables, DebugAdapterTracker {
     private refreshEventEmitter = new EventEmitter<void>();
     private lastKnownVariables: IJupyterVariable[] = [];
-    private importedIntoKernel = new Set<string>();
+    private importedDataFrameScriptsIntoKernel = new Set<string>();
+    private importedGetVariableInfoScriptsIntoKernel = new Set<string>();
     private watchedNotebooks = new Map<string, Disposable[]>();
     private debuggingStarted = false;
     constructor(
@@ -109,7 +110,7 @@ export class DebuggerVariables extends DebugLocationTracker
 
         // Then eval calling the main function with our target variable
         const results = await this.evaluate(
-            `${DataFrameLoading.DataFrameInfoFunc}(${targetVariable.name})`,
+            `${DataFrameLoading.DataFrameInfoImportFunc}(${targetVariable.name})`,
             // tslint:disable-next-line: no-any
             (targetVariable as any).frameId
         );
@@ -154,7 +155,7 @@ export class DebuggerVariables extends DebugLocationTracker
         for (let pos = start; pos < end; pos += chunkSize) {
             const chunkEnd = Math.min(pos + chunkSize, minnedEnd);
             const results = await this.evaluate(
-                `${DataFrameLoading.DataFrameRowFunc}(${targetVariable.name}, ${pos}, ${chunkEnd})`,
+                `${DataFrameLoading.DataFrameRowImportFunc}(${targetVariable.name}, ${pos}, ${chunkEnd})`,
                 // tslint:disable-next-line: no-any
                 (targetVariable as any).frameId
             );
@@ -194,7 +195,8 @@ export class DebuggerVariables extends DebugLocationTracker
             this.refreshEventEmitter.fire();
             const key = this.debugService.activeDebugSession?.id;
             if (key) {
-                this.importedIntoKernel.delete(key);
+                this.importedDataFrameScriptsIntoKernel.delete(key);
+                this.importedGetVariableInfoScriptsIntoKernel.delete(key);
             }
         }
     }
@@ -217,7 +219,8 @@ export class DebuggerVariables extends DebugLocationTracker
     }
 
     private resetImport(key: string) {
-        this.importedIntoKernel.delete(key);
+        this.importedDataFrameScriptsIntoKernel.delete(key);
+        this.importedGetVariableInfoScriptsIntoKernel.delete(key);
     }
 
     // tslint:disable-next-line: no-any
@@ -243,12 +246,24 @@ export class DebuggerVariables extends DebugLocationTracker
         try {
             // Run our dataframe scripts only once per session because they're slow
             const key = this.debugService.activeDebugSession?.id;
-            if (key && !this.importedIntoKernel.has(key)) {
+            if (key && !this.importedDataFrameScriptsIntoKernel.has(key)) {
                 await this.evaluate(DataFrameLoading.DataFrameSysImport);
-                await this.evaluate(DataFrameLoading.DataFrameInfoImport);
-                await this.evaluate(DataFrameLoading.DataFrameRowImport);
-                await this.evaluate(DataFrameLoading.VariableInfoImport);
-                this.importedIntoKernel.add(key);
+                await this.evaluate(DataFrameLoading.DataFrameImport);
+                this.importedDataFrameScriptsIntoKernel.add(key);
+            }
+        } catch (exc) {
+            traceError('Error attempting to import in debugger', exc);
+        }
+    }
+
+    private async importGetVariableInfoScripts(): Promise<void> {
+        try {
+            // Run our variable info scripts only once per session because they're slow
+            const key = this.debugService.activeDebugSession?.id;
+            if (key && !this.importedGetVariableInfoScriptsIntoKernel.has(key)) {
+                await this.evaluate(GetVariableInfo.GetVariableInfoSysImport);
+                await this.evaluate(GetVariableInfo.VariableInfoImport);
+                this.importedGetVariableInfoScriptsIntoKernel.add(key);
             }
         } catch (exc) {
             traceError('Error attempting to import in debugger', exc);
@@ -257,11 +272,11 @@ export class DebuggerVariables extends DebugLocationTracker
 
     private async getFullVariable(variable: IJupyterVariable): Promise<IJupyterVariable> {
         // See if we imported or not into the kernel our special function
-        await this.importDataFrameScripts();
+        await this.importGetVariableInfoScripts();
 
         // Then eval calling the variable info function with our target variable
         const results = await this.evaluate(
-            `${DataFrameLoading.VariableInfoFunc}(${variable.name})`,
+            `${GetVariableInfo.VariableInfoImportFunc}(${variable.name})`,
             // tslint:disable-next-line: no-any
             (variable as any).frameId
         );