diff --git a/README.md b/README.md index b841266..01bf43d 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,8 @@ cities.query({name: 'York'}).then(display); ### store +** This function is deprecated as Node.js global variables are copied to the Python environment automatically ** + ```js %%node @@ -125,7 +127,7 @@ var str = 'Sales are up 25%'; html(str); ``` -### js +### image ```js %%node @@ -140,6 +142,69 @@ image(url); help(); ``` +## Node.js-Python bridge + +Any *global* variables that you create in your `%%node` cells will be automatically copied to equivalent variables in Python. e.g if you create some variables in a Node.js cell: + +``` +%%node +var str = "hello world"; +var n1 = 4.1515; +var n2 = 42; +var tf = true; +var obj = { name:"Frank", age: 42 }; +var array_of_strings = ["hello", "world"]; +var array_of_objects = [{a:1,b:2}, {a:3, b:4}]; +``` + +Then these variables can be used in Python: + +``` +# Python cell +print str, n1, n2, tf +print obj +print array_of_strings +print array_of_objects +``` + +Strings, numbers, booleans and arrays of such are converted to their equivalent in Python. Objects are converted into Python dictionaries and arrays of objects are automatically converted into a Pandas DataFrames. + +Note that only variables declared with `var` are moved to Python, not constants declared with `const`. + + +If you want to move data from an asynchronous Node.js callback, remember to write it to a *global variable*: + +```js +%%node +var googlehomepage = ''; +request.get('http://www.google.com').then(function(data) { + googlehomepage = data; + print('Fetched Google homepage'); +}); +``` + +Similarly, Python variables of type `str`, `int`, `float`, `bool`, `unicode`, `dict` or `list` will be moved to Node.js when a cell is executed: + +``` +# Python cell +a = 'hello' +b = 2 +b = 3 +c= False +d = {} +d["x"] = 1 +d["y"] = 2 +e = 3.142 +``` + +The variables can then be used in Node.js: + +``` +%%node +console.log(a,b,c,d,e); +// hello 3 false { y: 2, x: 1 } 3.142 +``` + ## Managing the Node.js process If enter some invalid syntax into a `%%node` cell, such as code with more opening brackets than closing brackes, then the Node.js interpreter may not think you have finished typing and you receive no output. diff --git a/pixiedust_node/node.py b/pixiedust_node/node.py index 366756f..a113174 100644 --- a/pixiedust_node/node.py +++ b/pixiedust_node/node.py @@ -13,6 +13,33 @@ from pixiedust.utils.environment import Environment from pixiedust.utils.shellAccess import ShellAccess +class VarWatcher(object): + """ + this class watches for cell "post_execute" events. When one occurs, it examines + the IPython shell for variables that have been set (only numbers and strings). + New or changed variables are moved over to the JavaScript environment. + """ + + def __init__(self, ip, ps): + self.shell = ip + self.ps = ps + ip.events.register('post_execute', self.post_execute) + self.clearCache() + + def clearCache(self): + self.cache = {} + + def post_execute(self): + for key in self.shell.user_ns: + v = self.shell.user_ns[key] + t = type(v) + # if this is one of our varables, is a number or a string or a float + if not key.startswith('_') and (t in (str, int, float, bool, unicode, dict, list)): + # if it's not in our cache or it is an its value has changed + if not key in self.cache or (key in self.cache and self.cache[key] != v): + # move it to JavaScript land and add it to our cache + self.ps.stdin.write("var " + key + " = " + json.dumps(v) + ";\r\n") + self.cache[key] = v class NodeStdReader(Thread): """ @@ -34,6 +61,7 @@ def stop(self): self._stop_event.set() def run(self): + # forever while not self._stop_event.is_set(): # read line from Node's stdout @@ -60,6 +88,7 @@ def run(self): elif obj['type'] == 'print': print(json.dumps(obj['data'])) elif obj['type'] == 'store': + print '!!! Warning: store is now deprecated - Node.js global variables are automatically propagated to Python !!!' variable = 'pdf' if 'variable' in obj: variable = obj['variable'] @@ -68,6 +97,9 @@ def run(self): IPython.display.display(IPython.display.HTML(obj['data'])) elif obj['type'] == 'image': IPython.display.display(IPython.display.HTML(''.format(obj['data']))) + elif obj['type'] == 'variable': + ShellAccess[obj['key']] = obj['value'] + except Exception as e: print(line) @@ -162,6 +194,9 @@ def __init__(self, path): # create thread to read this process's output NodeStdReader(self.ps) + # watch Python variables for changes + self.vw = VarWatcher(get_ipython(), self.ps) + def write(self, s): self.ps.stdin.write(s) self.ps.stdin.write("\r\n") @@ -172,6 +207,7 @@ def cancel(self): def clear(self): self.write("\r\n.clear") + self.vw.clearCache() def help(self): self.cancel() @@ -217,3 +253,6 @@ def uninstall(self, module): def list(self): self.cmd('list', None) + + + diff --git a/pixiedust_node/pixiedustNodeRepl.js b/pixiedust_node/pixiedustNodeRepl.js index 31b550d..50b40f5 100644 --- a/pixiedust_node/pixiedustNodeRepl.js +++ b/pixiedust_node/pixiedustNodeRepl.js @@ -1,13 +1,42 @@ const repl = require('repl'); const pkg = require('./package.json'); +const crypto = require('crypto'); + -// custom writer function that outputs nothing -const writer = function(output) { - // don't output anything - return ''; -}; const startRepl = function(instream, outstream) { + + // check for Node.js global variables and move those values to Python + const globalVariableChecker = function() { + var varlist = Object.getOwnPropertyNames(r.context); + const cutoff = varlist.indexOf('help') + 1; + varlist.splice(0, cutoff); + if (varlist.length === 0) return; + for(var i in varlist) { + const v = varlist[i]; + const j = JSON.stringify(r.context[v]); + if (typeof j === 'string' ) { + const h = hash(j); + if (lastGlobal[v] !== h) { + const datatype = isArray(r.context[v]) && typeof r.context[v][0] === 'object' ? 'array' : typeof r.context[v]; + const obj = { _pixiedust: true, type: 'variable', key: v, datatype: datatype, value: r.context[v] }; + outstream.write('\n' + JSON.stringify(obj) + '\n') + lastGlobal[v] = h; + } + } + } + }; + + // sync Node.js to Python every 1 second + interval = setInterval(globalVariableChecker, 1000); + + // custom writer function that outputs nothing + const writer = function(output) { + globalVariableChecker(); + // don't output anything + return ''; + }; + const options = { input: instream, output: outstream, @@ -15,40 +44,59 @@ const startRepl = function(instream, outstream) { writer: writer }; const r = repl.start(options); + var lastGlobal = {}; + var interval = null; + + // generate hash from data + const hash = function(data) { + return crypto.createHash('md5').update(data).digest("hex"); + } + + const isArray = Array.isArray || function(obj) { + return obj && toString.call(obj) === '[object Array]'; + }; // custom print function for Notebook interface const print = function(data) { // bundle the data into an object + globalVariableChecker(); const obj = { _pixiedust: true, type: 'print', data: data }; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write(JSON.stringify(obj) + '\n'); }; // custom display function for Notebook interface const display = function(data) { // bundle the data into an object + globalVariableChecker(); const obj = { _pixiedust: true, type: 'display', data: data }; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write(JSON.stringify(obj) + '\n'); + }; // custom display function for Notebook interface const store = function(data, variable) { + globalVariableChecker(); + if (!data && !variable) return; // bundle the data into an object const obj = { _pixiedust: true, type: 'store', data: data, variable: variable }; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write(JSON.stringify(obj) + '\n'); + }; // display html in Notebook cell const html = function(data) { // bundle the data into an object const obj = { _pixiedust: true, type: 'html', data: data}; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write(JSON.stringify(obj) + '\n'); + globalVariableChecker(); }; // display image in Notebook cell const image = function(data) { // bundle the data into an object const obj = { _pixiedust: true, type: 'image', data: data}; - outstream.write(JSON.stringify(obj) + '\n') + outstream.write(JSON.stringify(obj) + '\n'); + globalVariableChecker(); }; const help = function() { @@ -58,7 +106,6 @@ const startRepl = function(instream, outstream) { console.log("JavaScript functions:"); console.log("* print(x) - print out x"); console.log("* display(x) - turn x into Pandas dataframe and display with Pixiedust"); - console.log("* store(x,'y') - turn x into Pandas dataframe and assign to Python variable y"); console.log("* html(x) - display HTML x in Notebook cell"); console.log("* image(x) - display image URL x in a Notebook cell"); console.log("* help() - display help"); @@ -80,6 +127,7 @@ const startRepl = function(instream, outstream) { r.context.html = html; r.context.image = image; r.context.help = help; + lastGlobal = {}; }; // add print/disply/store back in on reset