Merge pull request #18 from ibm-watson-data-lab/bridge

Bridge

README.md

@@ -102,6 +102,8 @@ cities.query({name: 'York'}).then(display);
 
 ### store
 
+** This function is deprecated as Node.js global variables are copied to the Python environment automatically **
+
 ```js
 %%node
 
@@ -125,7 +127,7 @@ var str = 'Sales are up <b>25%</b>';
 html(str);
 ```
 
-### js
+### image
 
 ```js
 %%node
@@ -140,6 +142,69 @@ image(url);
 help();
 ```
 
+## Node.js-Python bridge
+
+Any *global* variables that you create in your `%%node` cells will be automatically copied to equivalent variables in Python. e.g if you create some variables in a Node.js cell:
+
+```
+%%node
+var str = "hello world";
+var n1 = 4.1515;
+var n2 = 42;
+var tf = true;
+var obj = { name:"Frank", age: 42 };
+var array_of_strings = ["hello", "world"];
+var array_of_objects = [{a:1,b:2}, {a:3, b:4}];
+```
+
+Then these variables can be used in Python:
+
+```
+# Python cell
+print str, n1, n2, tf
+print obj
+print array_of_strings
+print array_of_objects
+```
+
+Strings, numbers, booleans and arrays of such are converted to their equivalent in Python. Objects are converted into Python dictionaries and arrays of objects are automatically converted into a Pandas DataFrames.
+
+Note that only variables declared with `var` are moved to Python, not constants declared with `const`.
+
+
+If you want to move data from an asynchronous Node.js callback, remember to write it to a *global variable*:
+
+```js
+%%node
+var googlehomepage = '';
+request.get('http://www.google.com').then(function(data) {
+  googlehomepage = data;
+  print('Fetched Google homepage');
+});
+```
+
+Similarly, Python variables of type `str`, `int`, `float`, `bool`, `unicode`, `dict` or `list` will be moved to Node.js when a cell is executed:
+
+```
+# Python cell
+a = 'hello'
+b = 2
+b = 3
+c= False
+d = {}
+d["x"] = 1
+d["y"] = 2
+e = 3.142
+```
+
+The variables can then be used in Node.js:
+
+```
+%%node
+console.log(a,b,c,d,e);
+// hello 3 false { y: 2, x: 1 } 3.142
+```
+
 ## Managing the Node.js process
 
 If enter some invalid syntax into a `%%node` cell, such as code with more opening brackets than closing brackes, then the Node.js interpreter may not think you have finished typing and you receive no output.

pixiedust_node/node.py

@@ -13,6 +13,33 @@
 from pixiedust.utils.environment import Environment
 from pixiedust.utils.shellAccess import ShellAccess
 
+class VarWatcher(object):
+    """
+    this class watches for cell "post_execute" events. When one occurs, it examines
+    the IPython shell for variables that have been set (only numbers and strings).
+    New or changed variables are moved over to the JavaScript environment.
+    """
+
+    def __init__(self, ip, ps):
+        self.shell = ip
+        self.ps = ps
+        ip.events.register('post_execute', self.post_execute)
+        self.clearCache()
+
+    def clearCache(self):
+        self.cache = {}
+
+    def post_execute(self):
+        for key in self.shell.user_ns:
+            v = self.shell.user_ns[key]
+            t = type(v)
+            # if this is one of our varables, is a number or a string or a float
+            if not key.startswith('_') and (t in (str, int, float, bool, unicode, dict, list)):
+                # if it's not in our cache or it is an its value has changed
+                if not key in self.cache or (key in self.cache and self.cache[key] != v):
+                    # move it to JavaScript land and add it to our cache
+                    self.ps.stdin.write("var " + key + " = " + json.dumps(v) + ";\r\n")
+                    self.cache[key] = v
 
 class NodeStdReader(Thread):
     """
@@ -34,6 +61,7 @@ def stop(self):
         self._stop_event.set()
 
     def run(self):
+
         # forever
         while not self._stop_event.is_set():
             # read line from Node's stdout
@@ -60,6 +88,7 @@ def run(self):
                     elif obj['type'] == 'print':
                         print(json.dumps(obj['data']))
                     elif obj['type'] == 'store':
+                        print '!!! Warning: store is now deprecated - Node.js global variables are automatically propagated to Python !!!'
                         variable = 'pdf'
                         if 'variable' in obj:
                             variable = obj['variable']
@@ -68,6 +97,9 @@ def run(self):
                         IPython.display.display(IPython.display.HTML(obj['data']))
                     elif obj['type'] == 'image':
                         IPython.display.display(IPython.display.HTML('<img src="{0}" />'.format(obj['data'])))
+                    elif obj['type'] == 'variable':
+                        ShellAccess[obj['key']] = obj['value']
+
 
             except Exception as e:
                 print(line)
@@ -162,6 +194,9 @@ def __init__(self, path):
         # create thread to read this process's output
         NodeStdReader(self.ps)
 
+        # watch Python variables for changes
+        self.vw = VarWatcher(get_ipython(), self.ps)
+
     def write(self, s):
         self.ps.stdin.write(s)
         self.ps.stdin.write("\r\n")
@@ -172,6 +207,7 @@ def cancel(self):
 
     def clear(self):
         self.write("\r\n.clear")
+        self.vw.clearCache()
 
     def help(self):
         self.cancel()
@@ -217,3 +253,6 @@ def uninstall(self, module):
 
     def list(self):
         self.cmd('list', None)
+
+
+

pixiedust_node/pixiedustNodeRepl.js

@@ -1,54 +1,102 @@
 const repl = require('repl');
 const pkg = require('./package.json');
+const crypto = require('crypto');
+
 
-// custom writer function that outputs nothing
-const writer = function(output) {
-  // don't output anything
-  return '';
-};
 
 const startRepl = function(instream, outstream) {
+
+  // check for Node.js global variables and move those values to Python
+  const globalVariableChecker = function() {
+    var varlist = Object.getOwnPropertyNames(r.context);
+    const cutoff = varlist.indexOf('help') + 1;
+    varlist.splice(0, cutoff);
+    if (varlist.length === 0) return;
+    for(var i in varlist) {
+      const v = varlist[i];
+      const j = JSON.stringify(r.context[v]);
+      if (typeof j === 'string' ) {
+        const h = hash(j);
+        if (lastGlobal[v] !== h) {
+          const datatype = isArray(r.context[v]) && typeof r.context[v][0] === 'object' ? 'array' : typeof r.context[v];
+          const obj = { _pixiedust: true, type: 'variable', key: v, datatype: datatype, value: r.context[v] };
+          outstream.write('\n' + JSON.stringify(obj) + '\n')
+          lastGlobal[v] = h;
+        }
+      }
+    }
+  };
+
+  // sync Node.js to Python every 1 second
+  interval = setInterval(globalVariableChecker, 1000);
+
+  // custom writer function that outputs nothing
+  const writer = function(output) {
+    globalVariableChecker();
+    // don't output anything
+    return '';
+  };
+
   const options = {
     input: instream,
     output: outstream,
     prompt: '',
     writer: writer
   };
   const r = repl.start(options);
+  var lastGlobal = {};
+  var interval = null;
+
+  // generate hash from data
+  const hash = function(data) {
+    return crypto.createHash('md5').update(data).digest("hex");
+  }
+
+  const isArray = Array.isArray || function(obj) {
+    return obj && toString.call(obj) === '[object Array]';
+  };
 
   // custom print function for Notebook interface
   const print = function(data) {
     // bundle the data into an object
+    globalVariableChecker();
     const obj = { _pixiedust: true, type: 'print', data: data };
-    outstream.write(JSON.stringify(obj) + '\n')
+    outstream.write(JSON.stringify(obj) + '\n');
   };
 
   // custom display function for Notebook interface
   const display = function(data) {
     // bundle the data into an object
+    globalVariableChecker();
     const obj = { _pixiedust: true, type: 'display', data: data };
-    outstream.write(JSON.stringify(obj) + '\n')
+    outstream.write(JSON.stringify(obj) + '\n');
+
   };
 
   // custom display function for Notebook interface
   const store = function(data, variable) {
+    globalVariableChecker();
+    if (!data && !variable) return;
     // bundle the data into an object
     const obj = { _pixiedust: true, type: 'store', data: data, variable: variable };
-    outstream.write(JSON.stringify(obj) + '\n')
+    outstream.write(JSON.stringify(obj) + '\n');
+
   };
 
   // display html in Notebook cell
   const html = function(data) {
     // bundle the data into an object
     const obj = { _pixiedust: true, type: 'html', data: data};
-    outstream.write(JSON.stringify(obj) + '\n')
+    outstream.write(JSON.stringify(obj) + '\n');
+    globalVariableChecker();
   };
 
   // display image in Notebook cell
   const image = function(data) {
     // bundle the data into an object
     const obj = { _pixiedust: true, type: 'image', data: data};
-    outstream.write(JSON.stringify(obj) + '\n')
+    outstream.write(JSON.stringify(obj) + '\n');
+    globalVariableChecker();
   };
 
   const help = function() {
@@ -58,7 +106,6 @@ const startRepl = function(instream, outstream) {
     console.log("JavaScript functions:");
     console.log("* print(x) - print out x");
     console.log("* display(x) - turn x into Pandas dataframe and display with Pixiedust");
-    console.log("* store(x,'y') - turn x into Pandas dataframe and assign to Python variable y");
     console.log("* html(x) - display HTML x in Notebook cell");
     console.log("* image(x) - display image URL x in a Notebook cell");
     console.log("* help() - display help");
@@ -80,6 +127,7 @@ const startRepl = function(instream, outstream) {
     r.context.html = html;
     r.context.image = image;
     r.context.help = help;
+    lastGlobal = {};
   };
 
   // add print/disply/store back in on reset