fix: Display HTML for IPython.display.display_html

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Next Release
 
+### Bug Fixes
+- Render the output of IPython.display.display_html as HTML
+
 ## 0.20.4
 
 ### Bug Fixes

sparkmagic/sparkmagic/magics/sparkmagicsbase.py

@@ -1,13 +1,17 @@
 # -*- coding: utf-8 -*-
 
-"""Runs Scala, PySpark and SQL statement through Spark using a REST endpoint in remote cluster.
-Provides the %spark magic."""
+"""Runs Scala, PySpark and SQL statement through Spark using a REST endpoint in
+remote cluster.
+
+Provides the %spark magic.
+"""
 
 # Copyright (c) 2015  aggftw@gmail.com
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 from six import string_types
+import json
 
 from IPython.core.magic import Magics, magics_class
 from hdijupyterutils.ipythondisplay import IpythonDisplay
@@ -32,6 +36,10 @@
 SparkOutputHandler = namedtuple("SparkOutputHandler", ["html", "text", "default"])
 
 
+def looks_like_json(s):
+    return s.startswith("{") and s.endswith("}")
+
+
 @magics_class
 class SparkMagicBase(Magics):
     _STRING_VAR_TYPE = "str"
@@ -136,7 +144,24 @@ def execute_spark(
                 if mimetype == MIMETYPE_TEXT_HTML:
                     output_handler.html(out)
                 else:
-                    output_handler.text(out)
+                    # Check for special case of { "text/html": "<div>...</div>" }
+                    # which is return by Livy from IPython display or display_html
+                    # parse out the html and display it
+                    if looks_like_json(out):
+                        try:
+                            # output will be in dict format (single quotes) so convert to JSON double quotes
+                            out_dict = json.loads(out.replace("'", '"'))
+                            if MIMETYPE_TEXT_HTML in out_dict:
+                                # display the html
+                                output_handler.html(out_dict[MIMETYPE_TEXT_HTML])
+                            else:
+                                # treat as text
+                                output_handler.text(out)
+                        except ValueError:
+                            # treat as text
+                            output_handler.text(out)
+                    else:
+                        output_handler.text(out)
             else:
                 output_handler.default(out)
             if output_var is not None:

sparkmagic/sparkmagic/tests/test_sparkmagicsbase.py

@@ -317,12 +317,29 @@ def test_spark_execution_with_output_var():
         )
 
 
+def test_spark_execution_ipython_display_html():
+    magic.spark_controller.run_command.return_value = (
+        True,
+        "{ 'text/html': '<h1>test</h1>' }",
+        MIMETYPE_TEXT_PLAIN,
+    )
+    magic.execute_spark(
+        "",
+        None,
+        None,
+        None,
+        None,
+        session,
+        True,
+    )
+    magic.ipython_display.html.assert_called_once_with("<h1>test</h1>")
+
+
 def test_spark_exception_with_output_var():
     mockSparkCommand = MagicMock()
     magic._spark_store_command = MagicMock(return_value=mockSparkCommand)
     exception = BadUserDataException("Ka-boom!")
     output_var = "var_name"
-    df = "df"
 
     magic.spark_controller.run_command.side_effect = [
         (True, "out", MIMETYPE_TEXT_PLAIN),