diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8ebf1a32..b7b5b7ea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,9 @@
## Next Release
+### Bug Fixes
+- Render the output of IPython.display.display_html as HTML
+
## 0.20.4
### Bug Fixes
diff --git a/sparkmagic/sparkmagic/magics/sparkmagicsbase.py b/sparkmagic/sparkmagic/magics/sparkmagicsbase.py
index 67f55c10..93fcc6f6 100644
--- a/sparkmagic/sparkmagic/magics/sparkmagicsbase.py
+++ b/sparkmagic/sparkmagic/magics/sparkmagicsbase.py
@@ -1,13 +1,17 @@
# -*- coding: utf-8 -*-
-"""Runs Scala, PySpark and SQL statement through Spark using a REST endpoint in remote cluster.
-Provides the %spark magic."""
+"""Runs Scala, PySpark and SQL statement through Spark using a REST endpoint in
+remote cluster.
+
+Provides the %spark magic.
+"""
# Copyright (c) 2015 aggftw@gmail.com
# Distributed under the terms of the Modified BSD License.
from __future__ import print_function
from six import string_types
+import json
from IPython.core.magic import Magics, magics_class
from hdijupyterutils.ipythondisplay import IpythonDisplay
@@ -32,6 +36,10 @@
SparkOutputHandler = namedtuple("SparkOutputHandler", ["html", "text", "default"])
+def looks_like_json(s):
+ return s.startswith("{") and s.endswith("}")
+
+
@magics_class
class SparkMagicBase(Magics):
_STRING_VAR_TYPE = "str"
@@ -136,7 +144,24 @@ def execute_spark(
if mimetype == MIMETYPE_TEXT_HTML:
output_handler.html(out)
else:
- output_handler.text(out)
+ # Check for special case of { "text/html": "
...
" }
+ # which is return by Livy from IPython display or display_html
+ # parse out the html and display it
+ if looks_like_json(out):
+ try:
+ # output will be in dict format (single quotes) so convert to JSON double quotes
+ out_dict = json.loads(out.replace("'", '"'))
+ if MIMETYPE_TEXT_HTML in out_dict:
+ # display the html
+ output_handler.html(out_dict[MIMETYPE_TEXT_HTML])
+ else:
+ # treat as text
+ output_handler.text(out)
+ except ValueError:
+ # treat as text
+ output_handler.text(out)
+ else:
+ output_handler.text(out)
else:
output_handler.default(out)
if output_var is not None:
diff --git a/sparkmagic/sparkmagic/tests/test_sparkmagicsbase.py b/sparkmagic/sparkmagic/tests/test_sparkmagicsbase.py
index d10a2015..846cb4ad 100644
--- a/sparkmagic/sparkmagic/tests/test_sparkmagicsbase.py
+++ b/sparkmagic/sparkmagic/tests/test_sparkmagicsbase.py
@@ -317,12 +317,29 @@ def test_spark_execution_with_output_var():
)
+def test_spark_execution_ipython_display_html():
+ magic.spark_controller.run_command.return_value = (
+ True,
+ "{ 'text/html': 'test
' }",
+ MIMETYPE_TEXT_PLAIN,
+ )
+ magic.execute_spark(
+ "",
+ None,
+ None,
+ None,
+ None,
+ session,
+ True,
+ )
+ magic.ipython_display.html.assert_called_once_with("test
")
+
+
def test_spark_exception_with_output_var():
mockSparkCommand = MagicMock()
magic._spark_store_command = MagicMock(return_value=mockSparkCommand)
exception = BadUserDataException("Ka-boom!")
output_var = "var_name"
- df = "df"
magic.spark_controller.run_command.side_effect = [
(True, "out", MIMETYPE_TEXT_PLAIN),