ZEPPELIN-1048: Pandas support for python interpreter

docs/interpreter/python.md

@@ -39,7 +39,7 @@ To access the help, type **help()**
 ## Python modules
 The interpreter can use all modules already installed (with pip, easy_install...)
 
-## Apply Zeppelin Dynamic Forms
+## Use Zeppelin Dynamic Forms
 You can leverage [Zeppelin Dynamic Form]({{BASE_PATH}}/manual/dynamicform.html) inside your Python code.
 
 **Zeppelin Dynamic Form can only be used if py4j Python library is installed in your system. If not, you can install it with `pip install py4j`.**
@@ -60,36 +60,47 @@ print("".join(z.checkbox("f3", [("o1","1"), ("o2","2")],["1"])))
 
 
 
-
 ## Zeppelin features not fully supported by the Python Interpreter
 
 * Interrupt a paragraph execution (`cancel()` method) is currently only supported in Linux and MacOs. If interpreter runs in another operating system (for instance MS Windows) , interrupt a paragraph will close the whole interpreter. A JIRA ticket ([ZEPPELIN-893](https://issues.apache.org/jira/browse/ZEPPELIN-893)) is opened to implement this feature in a next release of the interpreter.
 * Progression bar in webUI  (`getProgress()` method) is currently not implemented.
 * Code-completion is currently not implemented.
 
 ## Matplotlib integration
- The python interpreter can display matplotlib graph with the function `zeppelin_show()`.
+ The python interpreter can display matplotlib graph with the function `z.show()`.
  You need to have matplotlib module installed and a XServer running to use this functionality !
 
  ```python
 %python
 import matplotlib.pyplot as plt
 plt.figure()
 (.. ..)
-zeppelin_show(plt)
+z.show(plt)
 plt.close()
 ```
 zeppelin_show function can take optional parameters to adapt graph width and height
 
  ```python
 %python
-zeppelin_show(plt,width='50px')
-zeppelin_show(plt,height='150px')
+z.show(plt, width='50px')
+z.show(plt, height='150px')
 ```
 
 [![pythonmatplotlib](../interpreter/screenshots/pythonMatplotlib.png)](/docs/interpreter/screenshots/pythonMatplotlib.png)
 
 
+## Pandas integration
+[Zeppelin Display System]({{BASE_PATH}}/displaysystem/basicdisplaysystem.html#table) provides simple API to visualize data in Pandas DataFrames, same as in Matplotlib.
+
+Example:
+
+```python
+import pandas as pd
+rates = pd.read_csv("bank.csv", sep=";")
+z.show(rates)
+```
+
+
 ## Technical description
 
 For in-depth technical details on current implementation plese reffer [python/README.md](https://github.com/apache/zeppelin/blob/master/python/README.md)

python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java

@@ -134,7 +134,9 @@ public void close() {
   @Override
   public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) {
     this.context = contextInterpreter;
-
+    if (cmd == null || cmd.isEmpty()) {
+      return new InterpreterResult(Code.SUCCESS, "");
+    }
     String output = sendCommandToPython(cmd);
     return new InterpreterResult(Code.SUCCESS, output.replaceAll(">>>", "")
         .replaceAll("\\.\\.\\.", "").trim());
@@ -193,12 +195,13 @@ private Job getRunningJob(String paragraphId) {
 
   private String sendCommandToPython(String cmd) {
     String output = "";
-    logger.info("Sending : \n" + (cmd.length() > 200 ? cmd.substring(0, 120) + "..." : cmd));
+    logger.info("Sending : \n" + (cmd.length() > 200 ? cmd.substring(0, 200) + "..." : cmd));
     try {
       output = process.sendAndGetResult(cmd);
     } catch (IOException e) {
       logger.error("Error when sending commands to python process", e);
     }
+    //logger.info("Got : \n" + output);
     return output;
   }
 

python/src/main/resources/bootstrap.py

@@ -33,6 +33,7 @@ def intHandler(signum, frame):  # Set the signal handler
 
 signal.signal(signal.SIGINT, intHandler)
 
+
 def help():
     print ('%html')
     print ('<h2>Python Interpreter help</h2>')
@@ -56,48 +57,102 @@ def help():
            '("o2","2")],["1"])))</pre>')
     print ('<h3>Matplotlib graph</h3>')
     print ('<div>The interpreter can display matplotlib graph with ')
-    print ('the function zeppelin_show()</div>')
+    print ('the function z.show()</div>')
     print ('<div> You need to already have matplotlib module installed ')
     print ('to use this functionality !</div><br/>')
     print ('''<pre>import matplotlib.pyplot as plt
 plt.figure()
 (.. ..)
-zeppelin_show(plt)
+z.show(plt)
 plt.close()
 </pre>''')
-    print ('<div><br/> zeppelin_show function can take optional parameters ')
+    print ('<div><br/> z.show function can take optional parameters ')
     print ('to adapt graph width and height</div>')
     print ("<div><b>example </b>:")
-    print('''<pre>zeppelin_show(plt,width='50px')
-zeppelin_show(plt,height='150px') </pre></div>''')
-
-# Matplotlib show function
-def zeppelin_show(p, width="0", height="0"):
-    img = io.StringIO()
-    p.savefig(img, format='svg')
-    img.seek(0)
-    style = ""
-    if(width != "0"):
-        style += 'width:'+width
-    if(height != "0"):
-        if(len(style) != 0):
-            style += ","
-            style += 'height:'+height
-    print("%html <div style='" + style + "'>" + img.read() + "<div>")
+    print ('''<pre>z.show(plt,width='50px')
+z.show(plt,height='150px') </pre></div>''')
+    print ('<h3>Pandas DataFrame</h3>')
+    print """
+<div>The interpreter can visualize Pandas DataFrame
+with the function z.show()
+<pre>
+import pandas as pd
+df = pd.read_csv("bank.csv", sep=";")
+z.show(df)
+</pre></div>
+"""
 
-# If py4j is detected, these class will be override
-# with the implementation in bootstrap_input.py
-class PyZeppelinContext():
+class PyZeppelinContext(object):
+    """ If py4j is detected, these class will be override
+        with the implementation in bootstrap_input.py
+    """
     errorMsg = "You must install py4j Python module " \
                "(pip install py4j) to use Zeppelin dynamic forms features"
+
     def __init__(self, zc):
         self.z = zc
+        self.max_result = 1000
+
     def input(self, name, defaultValue=""):
         print (self.errorMsg)
+
     def select(self, name, options, defaultValue=""):
         print (self.errorMsg)
+
     def checkbox(self, name, options, defaultChecked=[]):
         print (self.errorMsg)
+
+    def show(self, p, **kwargs):
+        if hasattr(p, '__name__') and p.__name__ == "matplotlib.pyplot":
+            self.show_matplotlib(p, **kwargs)
+        elif type(p).__name__ == "DataFrame": # does not play well with sub-classes
+            # `isinstance(p, DataFrame)` would req `import pandas.core.frame.DataFrame`
+            # and so a dependency on pandas
+            self.show_dataframe(p, **kwargs)
+
+    def show_dataframe(self, df, **kwargs):
+        """Pretty prints DF using Table Display System
+        """
+        limit = len(df) > self.max_result
+        header_buf = io.StringIO("")
+        header_buf.write(df.columns[0])
+        for col in df.columns[1:]:
+            header_buf.write("\t")
+            header_buf.write(col)
+        header_buf.write("\n")
+
+        body_buf = io.StringIO("")
+        rows = df.head(self.max_result).values if limit else df.values
+        for row in rows:
+            body_buf.write(row[0])
+            for cell in row[1:]:
+                body_buf.write("\t")
+                body_buf.write(cell)
+            body_buf.write("\n")
+        body_buf.seek(0); header_buf.seek(0)
+        #TODO(bzz): fix it, so it shows red notice, as in Spark
+        print("%table " + header_buf.read() + body_buf.read()) # +
+        #      ("\n<font color=red>Results are limited by {}.</font>" \
+        #          .format(self.max_result) if limit else "")
+        #)
+        body_buf.close(); header_buf.close()
+
+    def show_matplotlib(self, p, width="0", height="0", **kwargs):
+        """Matplotlib show function
+        """
+        img = io.StringIO()
+        p.savefig(img, format='svg')
+        img.seek(0)
+        style = ""
+        if (width != "0"):
+            style += 'width:' + width
+        if (height != "0"):
+            if (len(style) != 0):
+                style += ","
+                style += 'height:' + height
+        print("%html <div style='" + style + "'>" + img.read() + "<div>")
+        img.close()
+
 
 z = PyZeppelinContext("")
 

python/src/main/resources/bootstrap_input.py

@@ -16,28 +16,32 @@
 from py4j.java_gateway import JavaGateway
 from py4j.java_gateway import java_import, JavaGateway, GatewayClient
 
+
 client = GatewayClient(port=%PORT%)
 gateway = JavaGateway(client)
 java_import(gateway.jvm, "org.apache.zeppelin.display.Input")
 
-class PyZeppelinContext():
-    paramOption = gateway.jvm.org.apache.zeppelin.display.Input.ParamOption
-    javaList = gateway.jvm.java.util.ArrayList
 
+class Py4jZeppelinContext(PyZeppelinContext):
+    """A context impl that uses Py4j to communicate to JVM
+    """
     def __init__(self, zc):
-        self.z = zc
-
+        super(Py4jZeppelinContext, self).__init__(zc)
+        self.paramOption = gateway.jvm.org.apache.zeppelin.display.Input.ParamOption
+        self.javaList = gateway.jvm.java.util.ArrayList
+        self.max_result = 1000 #TODO(bzz): read `zeppelin.python.maxResult` from JVM
+
     def input(self, name, defaultValue=""):
         return self.z.getGui().input(name, defaultValue)
-
+    
     def select(self, name, options, defaultValue=""):
         javaOptions = gateway.new_array(self.paramOption, len(options))
         i = 0
         for tuple in options:
             javaOptions[i] = self.paramOption(tuple[0], tuple[1])
             i += 1
         return self.z.getGui().select(name, defaultValue, javaOptions)
-
+    
     def checkbox(self, name, options, defaultChecked=[]):
         javaOptions = gateway.new_array(self.paramOption, len(options))
         i = 0
@@ -49,4 +53,5 @@ def checkbox(self, name, options, defaultChecked=[]):
             javaDefaultCheck.append(check)
         return self.z.getGui().checkbox(name, javaDefaultCheck, javaOptions)
 
-z = PyZeppelinContext(gateway.entry_point)
+
+z = Py4jZeppelinContext(gateway.entry_point)