diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ebf1a32..b7b5b7ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Next Release +### Bug Fixes +- Render the output of IPython.display.display_html as HTML + ## 0.20.4 ### Bug Fixes diff --git a/sparkmagic/sparkmagic/magics/sparkmagicsbase.py b/sparkmagic/sparkmagic/magics/sparkmagicsbase.py index 67f55c10..93fcc6f6 100644 --- a/sparkmagic/sparkmagic/magics/sparkmagicsbase.py +++ b/sparkmagic/sparkmagic/magics/sparkmagicsbase.py @@ -1,13 +1,17 @@ # -*- coding: utf-8 -*- -"""Runs Scala, PySpark and SQL statement through Spark using a REST endpoint in remote cluster. -Provides the %spark magic.""" +"""Runs Scala, PySpark and SQL statement through Spark using a REST endpoint in +remote cluster. + +Provides the %spark magic. +""" # Copyright (c) 2015 aggftw@gmail.com # Distributed under the terms of the Modified BSD License. from __future__ import print_function from six import string_types +import json from IPython.core.magic import Magics, magics_class from hdijupyterutils.ipythondisplay import IpythonDisplay @@ -32,6 +36,10 @@ SparkOutputHandler = namedtuple("SparkOutputHandler", ["html", "text", "default"]) +def looks_like_json(s): + return s.startswith("{") and s.endswith("}") + + @magics_class class SparkMagicBase(Magics): _STRING_VAR_TYPE = "str" @@ -136,7 +144,24 @@ def execute_spark( if mimetype == MIMETYPE_TEXT_HTML: output_handler.html(out) else: - output_handler.text(out) + # Check for special case of { "text/html": "
...
" } + # which is return by Livy from IPython display or display_html + # parse out the html and display it + if looks_like_json(out): + try: + # output will be in dict format (single quotes) so convert to JSON double quotes + out_dict = json.loads(out.replace("'", '"')) + if MIMETYPE_TEXT_HTML in out_dict: + # display the html + output_handler.html(out_dict[MIMETYPE_TEXT_HTML]) + else: + # treat as text + output_handler.text(out) + except ValueError: + # treat as text + output_handler.text(out) + else: + output_handler.text(out) else: output_handler.default(out) if output_var is not None: diff --git a/sparkmagic/sparkmagic/tests/test_sparkmagicsbase.py b/sparkmagic/sparkmagic/tests/test_sparkmagicsbase.py index d10a2015..846cb4ad 100644 --- a/sparkmagic/sparkmagic/tests/test_sparkmagicsbase.py +++ b/sparkmagic/sparkmagic/tests/test_sparkmagicsbase.py @@ -317,12 +317,29 @@ def test_spark_execution_with_output_var(): ) +def test_spark_execution_ipython_display_html(): + magic.spark_controller.run_command.return_value = ( + True, + "{ 'text/html': '

test

' }", + MIMETYPE_TEXT_PLAIN, + ) + magic.execute_spark( + "", + None, + None, + None, + None, + session, + True, + ) + magic.ipython_display.html.assert_called_once_with("

test

") + + def test_spark_exception_with_output_var(): mockSparkCommand = MagicMock() magic._spark_store_command = MagicMock(return_value=mockSparkCommand) exception = BadUserDataException("Ka-boom!") output_var = "var_name" - df = "df" magic.spark_controller.run_command.side_effect = [ (True, "out", MIMETYPE_TEXT_PLAIN),