diff --git a/README.md b/README.md
index b841266..01bf43d 100644
--- a/README.md
+++ b/README.md
@@ -102,6 +102,8 @@ cities.query({name: 'York'}).then(display);
### store
+** This function is deprecated as Node.js global variables are copied to the Python environment automatically **
+
```js
%%node
@@ -125,7 +127,7 @@ var str = 'Sales are up 25%';
html(str);
```
-### js
+### image
```js
%%node
@@ -140,6 +142,69 @@ image(url);
help();
```
+## Node.js-Python bridge
+
+Any *global* variables that you create in your `%%node` cells will be automatically copied to equivalent variables in Python. e.g if you create some variables in a Node.js cell:
+
+```
+%%node
+var str = "hello world";
+var n1 = 4.1515;
+var n2 = 42;
+var tf = true;
+var obj = { name:"Frank", age: 42 };
+var array_of_strings = ["hello", "world"];
+var array_of_objects = [{a:1,b:2}, {a:3, b:4}];
+```
+
+Then these variables can be used in Python:
+
+```
+# Python cell
+print str, n1, n2, tf
+print obj
+print array_of_strings
+print array_of_objects
+```
+
+Strings, numbers, booleans and arrays of such are converted to their equivalent in Python. Objects are converted into Python dictionaries and arrays of objects are automatically converted into a Pandas DataFrames.
+
+Note that only variables declared with `var` are moved to Python, not constants declared with `const`.
+
+
+If you want to move data from an asynchronous Node.js callback, remember to write it to a *global variable*:
+
+```js
+%%node
+var googlehomepage = '';
+request.get('http://www.google.com').then(function(data) {
+ googlehomepage = data;
+ print('Fetched Google homepage');
+});
+```
+
+Similarly, Python variables of type `str`, `int`, `float`, `bool`, `unicode`, `dict` or `list` will be moved to Node.js when a cell is executed:
+
+```
+# Python cell
+a = 'hello'
+b = 2
+b = 3
+c= False
+d = {}
+d["x"] = 1
+d["y"] = 2
+e = 3.142
+```
+
+The variables can then be used in Node.js:
+
+```
+%%node
+console.log(a,b,c,d,e);
+// hello 3 false { y: 2, x: 1 } 3.142
+```
+
## Managing the Node.js process
If enter some invalid syntax into a `%%node` cell, such as code with more opening brackets than closing brackes, then the Node.js interpreter may not think you have finished typing and you receive no output.
diff --git a/pixiedust_node/node.py b/pixiedust_node/node.py
index 366756f..a113174 100644
--- a/pixiedust_node/node.py
+++ b/pixiedust_node/node.py
@@ -13,6 +13,33 @@
from pixiedust.utils.environment import Environment
from pixiedust.utils.shellAccess import ShellAccess
+class VarWatcher(object):
+ """
+ this class watches for cell "post_execute" events. When one occurs, it examines
+ the IPython shell for variables that have been set (only numbers and strings).
+ New or changed variables are moved over to the JavaScript environment.
+ """
+
+ def __init__(self, ip, ps):
+ self.shell = ip
+ self.ps = ps
+ ip.events.register('post_execute', self.post_execute)
+ self.clearCache()
+
+ def clearCache(self):
+ self.cache = {}
+
+ def post_execute(self):
+ for key in self.shell.user_ns:
+ v = self.shell.user_ns[key]
+ t = type(v)
+ # if this is one of our varables, is a number or a string or a float
+ if not key.startswith('_') and (t in (str, int, float, bool, unicode, dict, list)):
+ # if it's not in our cache or it is an its value has changed
+ if not key in self.cache or (key in self.cache and self.cache[key] != v):
+ # move it to JavaScript land and add it to our cache
+ self.ps.stdin.write("var " + key + " = " + json.dumps(v) + ";\r\n")
+ self.cache[key] = v
class NodeStdReader(Thread):
"""
@@ -34,6 +61,7 @@ def stop(self):
self._stop_event.set()
def run(self):
+
# forever
while not self._stop_event.is_set():
# read line from Node's stdout
@@ -60,6 +88,7 @@ def run(self):
elif obj['type'] == 'print':
print(json.dumps(obj['data']))
elif obj['type'] == 'store':
+ print '!!! Warning: store is now deprecated - Node.js global variables are automatically propagated to Python !!!'
variable = 'pdf'
if 'variable' in obj:
variable = obj['variable']
@@ -68,6 +97,9 @@ def run(self):
IPython.display.display(IPython.display.HTML(obj['data']))
elif obj['type'] == 'image':
IPython.display.display(IPython.display.HTML(''.format(obj['data'])))
+ elif obj['type'] == 'variable':
+ ShellAccess[obj['key']] = obj['value']
+
except Exception as e:
print(line)
@@ -162,6 +194,9 @@ def __init__(self, path):
# create thread to read this process's output
NodeStdReader(self.ps)
+ # watch Python variables for changes
+ self.vw = VarWatcher(get_ipython(), self.ps)
+
def write(self, s):
self.ps.stdin.write(s)
self.ps.stdin.write("\r\n")
@@ -172,6 +207,7 @@ def cancel(self):
def clear(self):
self.write("\r\n.clear")
+ self.vw.clearCache()
def help(self):
self.cancel()
@@ -217,3 +253,6 @@ def uninstall(self, module):
def list(self):
self.cmd('list', None)
+
+
+
diff --git a/pixiedust_node/pixiedustNodeRepl.js b/pixiedust_node/pixiedustNodeRepl.js
index 31b550d..50b40f5 100644
--- a/pixiedust_node/pixiedustNodeRepl.js
+++ b/pixiedust_node/pixiedustNodeRepl.js
@@ -1,13 +1,42 @@
const repl = require('repl');
const pkg = require('./package.json');
+const crypto = require('crypto');
+
-// custom writer function that outputs nothing
-const writer = function(output) {
- // don't output anything
- return '';
-};
const startRepl = function(instream, outstream) {
+
+ // check for Node.js global variables and move those values to Python
+ const globalVariableChecker = function() {
+ var varlist = Object.getOwnPropertyNames(r.context);
+ const cutoff = varlist.indexOf('help') + 1;
+ varlist.splice(0, cutoff);
+ if (varlist.length === 0) return;
+ for(var i in varlist) {
+ const v = varlist[i];
+ const j = JSON.stringify(r.context[v]);
+ if (typeof j === 'string' ) {
+ const h = hash(j);
+ if (lastGlobal[v] !== h) {
+ const datatype = isArray(r.context[v]) && typeof r.context[v][0] === 'object' ? 'array' : typeof r.context[v];
+ const obj = { _pixiedust: true, type: 'variable', key: v, datatype: datatype, value: r.context[v] };
+ outstream.write('\n' + JSON.stringify(obj) + '\n')
+ lastGlobal[v] = h;
+ }
+ }
+ }
+ };
+
+ // sync Node.js to Python every 1 second
+ interval = setInterval(globalVariableChecker, 1000);
+
+ // custom writer function that outputs nothing
+ const writer = function(output) {
+ globalVariableChecker();
+ // don't output anything
+ return '';
+ };
+
const options = {
input: instream,
output: outstream,
@@ -15,40 +44,59 @@ const startRepl = function(instream, outstream) {
writer: writer
};
const r = repl.start(options);
+ var lastGlobal = {};
+ var interval = null;
+
+ // generate hash from data
+ const hash = function(data) {
+ return crypto.createHash('md5').update(data).digest("hex");
+ }
+
+ const isArray = Array.isArray || function(obj) {
+ return obj && toString.call(obj) === '[object Array]';
+ };
// custom print function for Notebook interface
const print = function(data) {
// bundle the data into an object
+ globalVariableChecker();
const obj = { _pixiedust: true, type: 'print', data: data };
- outstream.write(JSON.stringify(obj) + '\n')
+ outstream.write(JSON.stringify(obj) + '\n');
};
// custom display function for Notebook interface
const display = function(data) {
// bundle the data into an object
+ globalVariableChecker();
const obj = { _pixiedust: true, type: 'display', data: data };
- outstream.write(JSON.stringify(obj) + '\n')
+ outstream.write(JSON.stringify(obj) + '\n');
+
};
// custom display function for Notebook interface
const store = function(data, variable) {
+ globalVariableChecker();
+ if (!data && !variable) return;
// bundle the data into an object
const obj = { _pixiedust: true, type: 'store', data: data, variable: variable };
- outstream.write(JSON.stringify(obj) + '\n')
+ outstream.write(JSON.stringify(obj) + '\n');
+
};
// display html in Notebook cell
const html = function(data) {
// bundle the data into an object
const obj = { _pixiedust: true, type: 'html', data: data};
- outstream.write(JSON.stringify(obj) + '\n')
+ outstream.write(JSON.stringify(obj) + '\n');
+ globalVariableChecker();
};
// display image in Notebook cell
const image = function(data) {
// bundle the data into an object
const obj = { _pixiedust: true, type: 'image', data: data};
- outstream.write(JSON.stringify(obj) + '\n')
+ outstream.write(JSON.stringify(obj) + '\n');
+ globalVariableChecker();
};
const help = function() {
@@ -58,7 +106,6 @@ const startRepl = function(instream, outstream) {
console.log("JavaScript functions:");
console.log("* print(x) - print out x");
console.log("* display(x) - turn x into Pandas dataframe and display with Pixiedust");
- console.log("* store(x,'y') - turn x into Pandas dataframe and assign to Python variable y");
console.log("* html(x) - display HTML x in Notebook cell");
console.log("* image(x) - display image URL x in a Notebook cell");
console.log("* help() - display help");
@@ -80,6 +127,7 @@ const startRepl = function(instream, outstream) {
r.context.html = html;
r.context.image = image;
r.context.help = help;
+ lastGlobal = {};
};
// add print/disply/store back in on reset