From 1b1964540920901362b8c8e5f4b247568d6dfdbd Mon Sep 17 00:00:00 2001 From: Glynn Bird Date: Tue, 16 Jan 2018 12:15:14 +0000 Subject: [PATCH 01/10] persist Node.js global variables into Python --- pixiedust_node/node.py | 7 ++++++ pixiedust_node/pixiedustNodeRepl.js | 34 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/pixiedust_node/node.py b/pixiedust_node/node.py index 366756f..8a5b986 100644 --- a/pixiedust_node/node.py +++ b/pixiedust_node/node.py @@ -60,6 +60,7 @@ def run(self): elif obj['type'] == 'print': print(json.dumps(obj['data'])) elif obj['type'] == 'store': + print '!!! Warning: store is now deprecated - Node.js global variables are automatically propagated to Python !!!' variable = 'pdf' if 'variable' in obj: variable = obj['variable'] @@ -68,6 +69,12 @@ def run(self): IPython.display.display(IPython.display.HTML(obj['data'])) elif obj['type'] == 'image': IPython.display.display(IPython.display.HTML(''.format(obj['data']))) + elif obj['type'] == 'variable': + if obj['datatype'] == 'array' : + ShellAccess[obj['key']] = pandas.DataFrame(obj['value']) + else: + ShellAccess[obj['key']] = obj['value'] + except Exception as e: print(line) diff --git a/pixiedust_node/pixiedustNodeRepl.js b/pixiedust_node/pixiedustNodeRepl.js index 31b550d..3bf046f 100644 --- a/pixiedust_node/pixiedustNodeRepl.js +++ b/pixiedust_node/pixiedustNodeRepl.js @@ -1,5 +1,6 @@ const repl = require('repl'); const pkg = require('./package.json'); +const crypto = require('crypto'); // custom writer function that outputs nothing const writer = function(output) { @@ -15,6 +16,39 @@ const startRepl = function(instream, outstream) { writer: writer }; const r = repl.start(options); + var lastGlobal = {}; + var interval = null; + + // generate hash from data + const hash = function(data) { + return crypto.createHash('md5').update(data).digest("hex"); + } + + const isArray = Array.isArray || function(obj) { + return obj && toString.call(obj) === '[object Array]'; + }; + + const globalVariableChecker = function() { + var varlist = Object.getOwnPropertyNames(r.context); + //console.log(varlist); + const cutoff = varlist.indexOf('help') + 1; + varlist.splice(0, cutoff); + //console.log(varlist); + if (varlist.length === 0) return; + for(var i in varlist) { + const v = varlist[i]; + const h = hash(JSON.stringify(r.context[v])); + if (lastGlobal[v] !== h) { + const datatype = isArray(r.context[v]) && typeof r.context[v][0] === 'object' ? 'array' : typeof r.context[v]; + const obj = { _pixiedust: true, type: 'variable', key: v, datatype: datatype, value: r.context[v] }; + outstream.write(JSON.stringify(obj) + '\n') + lastGlobal[v] = h; + } + } + }; + + // sync Node.js to Python every 1 second + interval = setInterval(globalVariableChecker, 1000); // custom print function for Notebook interface const print = function(data) { From c9c3524d324b6234d26eff0fabfd29264433024d Mon Sep 17 00:00:00 2001 From: Glynn Bird Date: Tue, 16 Jan 2018 13:45:41 +0000 Subject: [PATCH 02/10] added documentation about variable bridge --- README.md | 30 ++++++++++++++++++++++++++++- pixiedust_node/pixiedustNodeRepl.js | 1 + 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b841266..219d1c5 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,8 @@ cities.query({name: 'York'}).then(display); ### store +** This function is deprecated as Node.js global variables are copied to the Python environment automatically ** + ```js %%node @@ -125,7 +127,7 @@ var str = 'Sales are up 25%'; html(str); ``` -### js +### image ```js %%node @@ -140,6 +142,32 @@ image(url); help(); ``` +## Node.js-Python bridge + +Any *global* variables that you create in your `%%node` cells will be automatically copied to equivalent variables in Python. e.g if you create some variables in a Node.js cell: + +``` +%%node +var str = "hello world"; +var n1 = 4.1515; +var n2 = 42; +var tf = true; +var obj = { name:"Frank", age: 42 }; +var array_of_strings = ["hello", "world"]; +var array_of_objects = [{a:1,b:2}, {a:3, b:4}]; +``` + +Then these variables can be used in Python: + +``` +print str, n1, n2, tf +print obj +print array_of_strings +print array_of_objects +``` + +Strings, numbers, booleans and arrays of such are converted to their equivalent in Python. Objects are converted into Python dictionaries and arrays of objects are automatically converted into a Pandas DataFrames. + ## Managing the Node.js process If enter some invalid syntax into a `%%node` cell, such as code with more opening brackets than closing brackes, then the Node.js interpreter may not think you have finished typing and you receive no output. diff --git a/pixiedust_node/pixiedustNodeRepl.js b/pixiedust_node/pixiedustNodeRepl.js index 3bf046f..2dab695 100644 --- a/pixiedust_node/pixiedustNodeRepl.js +++ b/pixiedust_node/pixiedustNodeRepl.js @@ -114,6 +114,7 @@ const startRepl = function(instream, outstream) { r.context.html = html; r.context.image = image; r.context.help = help; + lastGlobal = {}; }; // add print/disply/store back in on reset From 5a8bd490e4dcbc2475445cab1f5a0db5b9a3bcaa Mon Sep 17 00:00:00 2001 From: Glynn Bird Date: Tue, 16 Jan 2018 13:47:20 +0000 Subject: [PATCH 03/10] additional documentation about const keyword --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 219d1c5..214ee1e 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,8 @@ print array_of_objects Strings, numbers, booleans and arrays of such are converted to their equivalent in Python. Objects are converted into Python dictionaries and arrays of objects are automatically converted into a Pandas DataFrames. +Note that only variables declared with `var` are moved to Python, not constants declared with `const`. + ## Managing the Node.js process If enter some invalid syntax into a `%%node` cell, such as code with more opening brackets than closing brackes, then the Node.js interpreter may not think you have finished typing and you receive no output. From ff48f786c0a4c44b540c320daa92c188c27133ab Mon Sep 17 00:00:00 2001 From: Glynn Bird Date: Wed, 17 Jan 2018 15:02:11 +0000 Subject: [PATCH 04/10] first stab at python->JS variable bridge --- pixiedust_node/node.py | 26 ++++++++-- pixiedust_node/pixiedustNodeRepl.js | 75 ++++++++++++++++------------- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/pixiedust_node/node.py b/pixiedust_node/node.py index 8a5b986..5dd9384 100644 --- a/pixiedust_node/node.py +++ b/pixiedust_node/node.py @@ -13,6 +13,18 @@ from pixiedust.utils.environment import Environment from pixiedust.utils.shellAccess import ShellAccess +class VarWatcher(object): + def __init__(self, ip, ps): + self.shell = ip + self.ps = ps + ip.events.register('post_execute', self.post_execute) + + def post_execute(self): + for key in self.shell.user_ns: + t = type(self.shell.user_ns[key]) + if not key.startswith('_') and (t == str or t == int): + #print("Setting JS variable: " + key + " from Python") + self.ps.stdin.write("var " + key + " = " + json.dumps(self.shell.user_ns[key]) + ";\r\n") class NodeStdReader(Thread): """ @@ -34,6 +46,7 @@ def stop(self): self._stop_event.set() def run(self): + # forever while not self._stop_event.is_set(): # read line from Node's stdout @@ -70,11 +83,8 @@ def run(self): elif obj['type'] == 'image': IPython.display.display(IPython.display.HTML(''.format(obj['data']))) elif obj['type'] == 'variable': - if obj['datatype'] == 'array' : - ShellAccess[obj['key']] = pandas.DataFrame(obj['value']) - else: - ShellAccess[obj['key']] = obj['value'] - + ShellAccess[obj['key']] = obj['value'] + except Exception as e: print(line) @@ -169,6 +179,9 @@ def __init__(self, path): # create thread to read this process's output NodeStdReader(self.ps) + # watch Python variables for changes + self.vw = VarWatcher(get_ipython(), self.ps) + def write(self, s): self.ps.stdin.write(s) self.ps.stdin.write("\r\n") @@ -224,3 +237,6 @@ def uninstall(self, module): def list(self): self.cmd('list', None) + + + diff --git a/pixiedust_node/pixiedustNodeRepl.js b/pixiedust_node/pixiedustNodeRepl.js index 2dab695..505a4fe 100644 --- a/pixiedust_node/pixiedustNodeRepl.js +++ b/pixiedust_node/pixiedustNodeRepl.js @@ -2,13 +2,38 @@ const repl = require('repl'); const pkg = require('./package.json'); const crypto = require('crypto'); -// custom writer function that outputs nothing -const writer = function(output) { - // don't output anything - return ''; -}; + const startRepl = function(instream, outstream) { + + // check for Node.js global variables and move those values to Python + const globalVariableChecker = function() { + var varlist = Object.getOwnPropertyNames(r.context); + const cutoff = varlist.indexOf('help') + 1; + varlist.splice(0, cutoff); + if (varlist.length === 0) return; + for(var i in varlist) { + const v = varlist[i]; + const h = hash(JSON.stringify(r.context[v])); + if (lastGlobal[v] !== h) { + const datatype = isArray(r.context[v]) && typeof r.context[v][0] === 'object' ? 'array' : typeof r.context[v]; + const obj = { _pixiedust: true, type: 'variable', key: v, datatype: datatype, value: r.context[v] }; + outstream.write(JSON.stringify(obj) + '\n') + lastGlobal[v] = h; + } + } + }; + + // sync Node.js to Python every 1 second + interval = setInterval(globalVariableChecker, 1000); + + // custom writer function that outputs nothing + const writer = function(output) { + globalVariableChecker(); + // don't output anything + return ''; + }; + const options = { input: instream, output: outstream, @@ -28,61 +53,46 @@ const startRepl = function(instream, outstream) { return obj && toString.call(obj) === '[object Array]'; }; - const globalVariableChecker = function() { - var varlist = Object.getOwnPropertyNames(r.context); - //console.log(varlist); - const cutoff = varlist.indexOf('help') + 1; - varlist.splice(0, cutoff); - //console.log(varlist); - if (varlist.length === 0) return; - for(var i in varlist) { - const v = varlist[i]; - const h = hash(JSON.stringify(r.context[v])); - if (lastGlobal[v] !== h) { - const datatype = isArray(r.context[v]) && typeof r.context[v][0] === 'object' ? 'array' : typeof r.context[v]; - const obj = { _pixiedust: true, type: 'variable', key: v, datatype: datatype, value: r.context[v] }; - outstream.write(JSON.stringify(obj) + '\n') - lastGlobal[v] = h; - } - } - }; - - // sync Node.js to Python every 1 second - interval = setInterval(globalVariableChecker, 1000); - // custom print function for Notebook interface const print = function(data) { // bundle the data into an object const obj = { _pixiedust: true, type: 'print', data: data }; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write(JSON.stringify(obj) + '\n'); + globalVariableChecker(); }; // custom display function for Notebook interface const display = function(data) { // bundle the data into an object const obj = { _pixiedust: true, type: 'display', data: data }; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write(JSON.stringify(obj) + '\n'); + globalVariableChecker(); }; // custom display function for Notebook interface const store = function(data, variable) { + globalVariableChecker(); + if (!data && !variable) return; // bundle the data into an object const obj = { _pixiedust: true, type: 'store', data: data, variable: variable }; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write(JSON.stringify(obj) + '\n'); + }; // display html in Notebook cell const html = function(data) { // bundle the data into an object const obj = { _pixiedust: true, type: 'html', data: data}; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write(JSON.stringify(obj) + '\n'); + globalVariableChecker(); }; // display image in Notebook cell const image = function(data) { // bundle the data into an object const obj = { _pixiedust: true, type: 'image', data: data}; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write(JSON.stringify(obj) + '\n'); + globalVariableChecker(); }; const help = function() { @@ -92,7 +102,6 @@ const startRepl = function(instream, outstream) { console.log("JavaScript functions:"); console.log("* print(x) - print out x"); console.log("* display(x) - turn x into Pandas dataframe and display with Pixiedust"); - console.log("* store(x,'y') - turn x into Pandas dataframe and assign to Python variable y"); console.log("* html(x) - display HTML x in Notebook cell"); console.log("* image(x) - display image URL x in a Notebook cell"); console.log("* help() - display help"); From c791bfa62a2fa4b011f8f84409ebd91a828b29ee Mon Sep 17 00:00:00 2001 From: Glynn Bird Date: Thu, 18 Jan 2018 10:37:52 +0000 Subject: [PATCH 05/10] added cache to only migrate variables that are new or that have changed --- pixiedust_node/node.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pixiedust_node/node.py b/pixiedust_node/node.py index 5dd9384..4652b31 100644 --- a/pixiedust_node/node.py +++ b/pixiedust_node/node.py @@ -18,13 +18,19 @@ def __init__(self, ip, ps): self.shell = ip self.ps = ps ip.events.register('post_execute', self.post_execute) + self.cache = {} def post_execute(self): for key in self.shell.user_ns: - t = type(self.shell.user_ns[key]) - if not key.startswith('_') and (t == str or t == int): - #print("Setting JS variable: " + key + " from Python") - self.ps.stdin.write("var " + key + " = " + json.dumps(self.shell.user_ns[key]) + ";\r\n") + v = self.shell.user_ns[key] + t = type(v) + # if this is one of our varables, is a number or a string or a float + if not key.startswith('_') and (t == str or t == int or t == unicode or t == float): + # if it's not in our cache or it is an its value has changed + if not key in self.cache or (key in self.cache and self.cache[key] != v): + # move it to JavaScript land and add it to our cache + self.ps.stdin.write("var " + key + " = " + json.dumps(v) + ";\r\n") + self.cache[key] = v class NodeStdReader(Thread): """ From 096a113fd0415744fa03d84d3d3a39ca521eef10 Mon Sep 17 00:00:00 2001 From: Glynn Bird Date: Thu, 18 Jan 2018 10:54:32 +0000 Subject: [PATCH 06/10] added comments --- pixiedust_node/node.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pixiedust_node/node.py b/pixiedust_node/node.py index 4652b31..64abf55 100644 --- a/pixiedust_node/node.py +++ b/pixiedust_node/node.py @@ -14,6 +14,12 @@ from pixiedust.utils.shellAccess import ShellAccess class VarWatcher(object): + """ + this class watches for cell "post_execute" events. When one occurs, it examines + the IPython shell for variables that have been set (only numbers and strings). + New or changed variables are moved over to the JavaScript environment. + """ + def __init__(self, ip, ps): self.shell = ip self.ps = ps From 9a7f9825b345ad486e963788e4f70d7f0a045e86 Mon Sep 17 00:00:00 2001 From: Glynn Bird Date: Thu, 18 Jan 2018 14:31:07 +0000 Subject: [PATCH 07/10] documentation on variable bridge --- README.md | 23 +++++++++++++++++++++++ pixiedust_node/node.py | 6 +++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 214ee1e..4ecab62 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,7 @@ var array_of_objects = [{a:1,b:2}, {a:3, b:4}]; Then these variables can be used in Python: ``` +# Python cell print str, n1, n2, tf print obj print array_of_strings @@ -170,6 +171,28 @@ Strings, numbers, booleans and arrays of such are converted to their equivalent Note that only variables declared with `var` are moved to Python, not constants declared with `const`. +Similarly, Python variables of type str, int, float, bool, unicode or dict will be moved to Node.js when a cell is executed: + +``` +# Python cell +a = 'hello' +b = 2 +b = 3 +c= False +d = {} +d["x"] = 1 +d["y"] = 2 +e = 3.142 +``` + +The variables can then be used in Node.js: + +``` +%%node +console.log(a,b,c,d,e); +// hello 3 false { y: 2, x: 1 } 3.142 +``` + ## Managing the Node.js process If enter some invalid syntax into a `%%node` cell, such as code with more opening brackets than closing brackes, then the Node.js interpreter may not think you have finished typing and you receive no output. diff --git a/pixiedust_node/node.py b/pixiedust_node/node.py index 64abf55..16aa255 100644 --- a/pixiedust_node/node.py +++ b/pixiedust_node/node.py @@ -24,6 +24,9 @@ def __init__(self, ip, ps): self.shell = ip self.ps = ps ip.events.register('post_execute', self.post_execute) + self.clearCache() + + def clearCache(self): self.cache = {} def post_execute(self): @@ -31,7 +34,7 @@ def post_execute(self): v = self.shell.user_ns[key] t = type(v) # if this is one of our varables, is a number or a string or a float - if not key.startswith('_') and (t == str or t == int or t == unicode or t == float): + if not key.startswith('_') and (t in (str, int, float, bool, unicode, dict)): # if it's not in our cache or it is an its value has changed if not key in self.cache or (key in self.cache and self.cache[key] != v): # move it to JavaScript land and add it to our cache @@ -204,6 +207,7 @@ def cancel(self): def clear(self): self.write("\r\n.clear") + self.vw.clearCache() def help(self): self.cancel() From 648a203099b6140f8fe8b2a8d9ef418e4063119b Mon Sep 17 00:00:00 2001 From: Glynn Bird Date: Thu, 18 Jan 2018 16:21:03 +0000 Subject: [PATCH 08/10] fix bug with variable bridge when the variable is a function --- README.md | 2 +- pixiedust_node/node.py | 2 +- pixiedust_node/pixiedustNodeRepl.js | 15 +++++++++------ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4ecab62..6104757 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ Strings, numbers, booleans and arrays of such are converted to their equivalent Note that only variables declared with `var` are moved to Python, not constants declared with `const`. -Similarly, Python variables of type str, int, float, bool, unicode or dict will be moved to Node.js when a cell is executed: +Similarly, Python variables of type `str`, `int`, `float`, `bool`, `unicode`, `dict` or `list` will be moved to Node.js when a cell is executed: ``` # Python cell diff --git a/pixiedust_node/node.py b/pixiedust_node/node.py index 16aa255..a113174 100644 --- a/pixiedust_node/node.py +++ b/pixiedust_node/node.py @@ -34,7 +34,7 @@ def post_execute(self): v = self.shell.user_ns[key] t = type(v) # if this is one of our varables, is a number or a string or a float - if not key.startswith('_') and (t in (str, int, float, bool, unicode, dict)): + if not key.startswith('_') and (t in (str, int, float, bool, unicode, dict, list)): # if it's not in our cache or it is an its value has changed if not key in self.cache or (key in self.cache and self.cache[key] != v): # move it to JavaScript land and add it to our cache diff --git a/pixiedust_node/pixiedustNodeRepl.js b/pixiedust_node/pixiedustNodeRepl.js index 505a4fe..25bbc16 100644 --- a/pixiedust_node/pixiedustNodeRepl.js +++ b/pixiedust_node/pixiedustNodeRepl.js @@ -14,12 +14,15 @@ const startRepl = function(instream, outstream) { if (varlist.length === 0) return; for(var i in varlist) { const v = varlist[i]; - const h = hash(JSON.stringify(r.context[v])); - if (lastGlobal[v] !== h) { - const datatype = isArray(r.context[v]) && typeof r.context[v][0] === 'object' ? 'array' : typeof r.context[v]; - const obj = { _pixiedust: true, type: 'variable', key: v, datatype: datatype, value: r.context[v] }; - outstream.write(JSON.stringify(obj) + '\n') - lastGlobal[v] = h; + const j = JSON.stringify(r.context[v]); + if (typeof j === 'string' ) { + const h = hash(j); + if (lastGlobal[v] !== h) { + const datatype = isArray(r.context[v]) && typeof r.context[v][0] === 'object' ? 'array' : typeof r.context[v]; + const obj = { _pixiedust: true, type: 'variable', key: v, datatype: datatype, value: r.context[v] }; + outstream.write(JSON.stringify(obj) + '\n') + lastGlobal[v] = h; + } } } }; From 936ab2fb55bdbf0d6cb32a4b0a9ac7f95e066383 Mon Sep 17 00:00:00 2001 From: Glynn Bird Date: Thu, 18 Jan 2018 16:50:14 +0000 Subject: [PATCH 09/10] fix bug in multi-line output --- pixiedust_node/pixiedustNodeRepl.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pixiedust_node/pixiedustNodeRepl.js b/pixiedust_node/pixiedustNodeRepl.js index 25bbc16..50b40f5 100644 --- a/pixiedust_node/pixiedustNodeRepl.js +++ b/pixiedust_node/pixiedustNodeRepl.js @@ -20,7 +20,7 @@ const startRepl = function(instream, outstream) { if (lastGlobal[v] !== h) { const datatype = isArray(r.context[v]) && typeof r.context[v][0] === 'object' ? 'array' : typeof r.context[v]; const obj = { _pixiedust: true, type: 'variable', key: v, datatype: datatype, value: r.context[v] }; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write('\n' + JSON.stringify(obj) + '\n') lastGlobal[v] = h; } } @@ -59,17 +59,18 @@ const startRepl = function(instream, outstream) { // custom print function for Notebook interface const print = function(data) { // bundle the data into an object + globalVariableChecker(); const obj = { _pixiedust: true, type: 'print', data: data }; outstream.write(JSON.stringify(obj) + '\n'); - globalVariableChecker(); }; // custom display function for Notebook interface const display = function(data) { // bundle the data into an object + globalVariableChecker(); const obj = { _pixiedust: true, type: 'display', data: data }; outstream.write(JSON.stringify(obj) + '\n'); - globalVariableChecker(); + }; // custom display function for Notebook interface From 4e93d4d33fb75aac2cb56a894f0ddbe529c2cc94 Mon Sep 17 00:00:00 2001 From: Glynn Bird Date: Fri, 19 Jan 2018 13:29:28 +0000 Subject: [PATCH 10/10] further documentation --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 6104757..01bf43d 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,18 @@ Strings, numbers, booleans and arrays of such are converted to their equivalent Note that only variables declared with `var` are moved to Python, not constants declared with `const`. + +If you want to move data from an asynchronous Node.js callback, remember to write it to a *global variable*: + +```js +%%node +var googlehomepage = ''; +request.get('http://www.google.com').then(function(data) { + googlehomepage = data; + print('Fetched Google homepage'); +}); +``` + Similarly, Python variables of type `str`, `int`, `float`, `bool`, `unicode`, `dict` or `list` will be moved to Node.js when a cell is executed: ```