From 38b05884fa97bfe4f9c51c8deeed08a8cc87bc09 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 18 Sep 2018 15:15:14 -0400 Subject: [PATCH 1/6] Added to/from/read/write json functions to plotly.io module --- plotly/io/__init__.py | 2 + plotly/io/_json.py | 193 +++++++++++++++++++ plotly/io/_orca.py | 15 +- plotly/io/_utils.py | 31 ++++ plotly/tests/test_io/test_to_from_json.py | 214 ++++++++++++++++++++++ tox.ini | 5 + 6 files changed, 447 insertions(+), 13 deletions(-) create mode 100644 plotly/io/_json.py create mode 100644 plotly/io/_utils.py create mode 100644 plotly/tests/test_io/test_to_from_json.py diff --git a/plotly/io/__init__.py b/plotly/io/__init__.py index 2162efb2d9..b02a7e8e4d 100644 --- a/plotly/io/__init__.py +++ b/plotly/io/__init__.py @@ -1,2 +1,4 @@ from ._orca import to_image, write_image from . import orca + +from ._json import to_json, from_json, read_json, write_json diff --git a/plotly/io/_json.py b/plotly/io/_json.py new file mode 100644 index 0000000000..0c4bda82ca --- /dev/null +++ b/plotly/io/_json.py @@ -0,0 +1,193 @@ +from six import string_types +import json + +from plotly.utils import PlotlyJSONEncoder +from plotly.io._utils import (validate_coerce_fig_to_dict, + validate_coerce_output_type) + + +def to_json(fig, + validate=True, + pretty=False, + remove_uids=True): + """ + Convert a figure to a JSON string representation + + Parameters + ---------- + fig: + Figure object or dict representing a figure + + validate: bool (default True) + True if the figure should be validated before being converted to + JSON, False otherwise. + + pretty: bool (default False) + True if JSON representation should be pretty-printed, False if + representation should be as compact as possible. + + remove_uids: bool (default True) + True if trace UIDs should be omitted from the JSON representation + + Returns + ------- + str + Representation of figure as a JSON string + """ + # Validate figure + # --------------- + fig_dict = validate_coerce_fig_to_dict(fig, validate) + + # Remove trace uid + # ---------------- + if remove_uids: + for trace in fig_dict.get('data', []): + trace.pop('uid') + + # Dump to a JSON string and return + # -------------------------------- + opts = {'sort_keys': True} + if pretty: + opts['indent'] = 2 + else: + # Remove all whitespace + opts['separators'] = (',', ':') + + return json.dumps(fig_dict, cls=PlotlyJSONEncoder, **opts) + + +def write_json(fig, file, validate=True, pretty=False, remove_uids=True): + """ + Convert a figure to JSON and write it to a file or writeable + object + + Parameters + ---------- + fig: + Figure object or dict representing a figure + + file: str or writeable + A string representing a local file path or a writeable object + (e.g. an open file descriptor) + + pretty: bool (default False) + True if JSON representation should be pretty-printed, False if + representation should be as compact as possible. + + remove_uids: bool (default True) + True if trace UIDs should be omitted from the JSON representation + + Returns + ------- + None + """ + + # Get JSON string + # --------------- + # Pass through validate argument and let to_json handle validation logic + json_str = to_json( + fig, validate=validate, pretty=pretty, remove_uids=remove_uids) + + # Check if file is a string + # ------------------------- + file_is_str = isinstance(file, string_types) + + # Open file + # --------- + if file_is_str: + with open(file, 'w') as f: + f.write(json_str) + else: + file.write(json_str) + + +def from_json(value, output_type='Figure', skip_invalid=False): + """ + Construct a figure from a JSON string + + Parameters + ---------- + value: str + String containing the JSON representation of a figure + output_type: type or str (default 'Figure') + The output figure type or type name. + One of: graph_objs.Figure, 'Figure, + graph_objs.FigureWidget, 'FigureWidget' + skip_invalid: bool (default False) + False if invalid figure properties should result in an exception + + Raises + ------ + ValueError + if value is not a string, or if skip_invalid=False and value contains + invalid figure properties + + Returns + ------- + Figure or FigureWidget + """ + + # Validate value + # -------------- + if not isinstance(value, string_types): + raise ValueError(""" +from_json requires a string argument but received value of type {typ} + Received value: {value}""".format(typ=type(value), + value=value)) + + # Decode JSON + # ----------- + fig_dict = json.loads(value) + + # Validate coerce output type + # --------------------------- + cls = validate_coerce_output_type(output_type) + + # Create and return figure + # ------------------------ + fig = cls(fig_dict, skip_invalid=skip_invalid) + return fig + + +def read_json(file, output_type='Figure', skip_invalid=False): + """ + Construct a figure from the JSON contents of a local file or readable + Python object + + Parameters + ---------- + file: str or readable + A string containing the path to a local file or a read-able Python + object (e.g. an open file descriptor) + + output_type: type or str (default 'Figure') + The output figure type or type name. + One of: graph_objs.Figure, 'Figure, + graph_objs.FigureWidget, 'FigureWidget' + skip_invalid: bool (default False) + False if invalid figure properties should result in an exception + + Returns + ------- + Figure or FigureWidget + """ + + # Check if file is a string + # ------------------------- + # If it's a string we assume it's a local file path. If it's not a string + # then we assume it's a read-able Python object + file_is_str = isinstance(file, string_types) + + # Read file contents into JSON string + # ----------------------------------- + if file_is_str: + with open(file, 'r') as f: + json_str = f.read() + else: + json_str = file.read() + + # Construct and return figure + # --------------------------- + return from_json(json_str, + skip_invalid=skip_invalid, + output_type=output_type) diff --git a/plotly/io/_orca.py b/plotly/io/_orca.py index 7a5f7b02d5..7b3acceb18 100644 --- a/plotly/io/_orca.py +++ b/plotly/io/_orca.py @@ -13,8 +13,8 @@ from six import string_types import plotly -from plotly.basedatatypes import BaseFigure from plotly.files import PLOTLY_DIR +from plotly.io._utils import validate_coerce_fig_to_dict from plotly.optional_imports import get_module psutil = get_module('psutil') @@ -1281,18 +1281,7 @@ def to_image(fig, # Validate figure # --------------- - if isinstance(fig, BaseFigure): - fig_dict = fig.to_plotly_json() - elif isinstance(fig, dict): - if validate: - # This will raise an exception if fig is not a valid plotly figure - fig_dict = plotly.graph_objs.Figure(fig).to_plotly_json() - else: - fig_dict = fig - else: - raise ValueError(""" -The fig parameter must be a dict or Figure. - Received value of type {typ}: {v}""".format(typ=type(fig), v=fig)) + fig_dict = validate_coerce_fig_to_dict(fig, validate) # Request image from server # ------------------------- diff --git a/plotly/io/_utils.py b/plotly/io/_utils.py new file mode 100644 index 0000000000..9b6ed825a3 --- /dev/null +++ b/plotly/io/_utils.py @@ -0,0 +1,31 @@ +import plotly +from plotly.basedatatypes import BaseFigure +from plotly.graph_objs import Figure, FigureWidget + + +def validate_coerce_fig_to_dict(fig, validate): + if isinstance(fig, BaseFigure): + fig_dict = fig.to_dict() + elif isinstance(fig, dict): + if validate: + # This will raise an exception if fig is not a valid plotly figure + fig_dict = plotly.graph_objs.Figure(fig).to_plotly_json() + else: + fig_dict = fig + else: + raise ValueError(""" +The fig parameter must be a dict or Figure. + Received value of type {typ}: {v}""".format(typ=type(fig), v=fig)) + return fig_dict + + +def validate_coerce_output_type(output_type): + if output_type == 'Figure' or output_type == Figure: + cls = Figure + elif output_type == 'FigureWidget' or output_type == FigureWidget: + cls = FigureWidget + else: + raise ValueError(""" +Invalid output type: {output_type} + Must be one of: 'Figure', 'FigureWidget'""") + return cls \ No newline at end of file diff --git a/plotly/tests/test_io/test_to_from_json.py b/plotly/tests/test_io/test_to_from_json.py new file mode 100644 index 0000000000..10c7991427 --- /dev/null +++ b/plotly/tests/test_io/test_to_from_json.py @@ -0,0 +1,214 @@ +import plotly.graph_objs as go +import plotly.io as pio +import pytest +import plotly +import json +import sys +import tempfile +import os + +if sys.version_info.major == 3 and sys.version_info.minor >= 3: + from unittest.mock import MagicMock +else: + from mock import MagicMock + +# fixtures +# -------- +@pytest.fixture +def fig1(request): + return go.Figure(data=[{'type': 'scattergl', + 'marker': {'color': 'green'}}, + {'type': 'parcoords', + 'dimensions': [{'values': [1, 2, 3]}, + {'values': [3, 2, 1]}], + 'line': {'color': 'blue'}}], + layout={'title': 'Figure title'}) + + +opts = {'separators': (',', ':'), + 'cls': plotly.utils.PlotlyJSONEncoder, + 'sort_keys': True} +pretty_opts = {'indent': 2, + 'cls': plotly.utils.PlotlyJSONEncoder, + 'sort_keys': True} + + +# to_json +# ------- +def test_to_json(fig1): + assert pio.to_json(fig1, remove_uids=False) == json.dumps( + fig1, **opts) + + +def test_to_json_remove_uids(fig1): + dict1 = fig1.to_dict() + for trace in dict1['data']: + trace.pop('uid', None) + + assert pio.to_json(fig1) == json.dumps( + dict1, **opts) + + +def test_to_json_validate(fig1): + dict1 = fig1.to_dict() + dict1['layout']['bogus'] = 37 + + with pytest.raises(ValueError): + pio.to_json(dict1) + + +def test_to_json_validate_false(fig1): + dict1 = fig1.to_dict() + dict1['layout']['bogus'] = 37 + + assert pio.to_json(dict1, validate=False) == json.dumps( + dict1, **opts) + + +def test_to_json_pretty_print(fig1): + assert pio.to_json(fig1, remove_uids=False, pretty=True) == json.dumps( + fig1, **pretty_opts) + + +# from_json +# --------- +def test_from_json(fig1): + fig1_json = json.dumps(fig1, **opts) + fig1_loaded = pio.from_json(fig1_json) + + # Check return type + assert isinstance(fig1_loaded, go.Figure) + + # Check return json + assert pio.to_json(fig1_loaded) == pio.to_json(fig1.to_dict()) + + +@pytest.mark.parametrize("fig_type_spec,fig_type", [ + ('Figure', go.Figure), + (go.Figure, go.Figure), + ('FigureWidget', go.FigureWidget), + (go.FigureWidget, go.FigureWidget) +]) +def test_from_json_output_type(fig1, fig_type_spec, fig_type): + fig1_json = json.dumps(fig1, **opts) + fig1_loaded = pio.from_json(fig1_json, output_type=fig_type_spec) + + # Check return type + assert isinstance(fig1_loaded, fig_type) + + # Check return json + assert pio.to_json(fig1_loaded) == pio.to_json(fig1.to_dict()) + + +def test_from_json_invalid(fig1): + dict1 = fig1.to_dict() + + # Set bad property name + dict1['data'][0]['marker']['bogus'] = 123 + + # Set property with bad value + dict1['data'][0]['marker']['size'] = -1 + + # Serialize to json + bad_json = json.dumps(dict1, **opts) + + with pytest.raises(ValueError): + pio.from_json(bad_json) + + +def test_from_json_skip_invalid(fig1): + dict1 = fig1.to_dict() + + # Set bad property name + dict1['data'][0]['marker']['bogus'] = 123 + + # Set property with bad value + dict1['data'][0]['marker']['size'] = -1 + + # Serialize to json + bad_json = json.dumps(dict1, **opts) + fig1_loaded = pio.from_json(bad_json, skip_invalid=True) + + # Check loaded figure + assert pio.to_json(fig1_loaded) == pio.to_json(fig1.to_dict()) + + +# read_json +# --------- +@pytest.mark.parametrize("fig_type_spec,fig_type", [ + ('Figure', go.Figure), + (go.Figure, go.Figure), + ('FigureWidget', go.FigureWidget), + (go.FigureWidget, go.FigureWidget) +]) +def test_read_json_from_filelike(fig1, fig_type_spec, fig_type): + # Configure file-like mock + filemock = MagicMock() + filemock.read.return_value = pio.to_json(fig1) + + # read_json on mock file + fig1_loaded = pio.read_json(filemock, output_type=fig_type_spec) + + # Check return type + assert isinstance(fig1_loaded, fig_type) + + # Check loaded figure + assert pio.to_json(fig1_loaded) == pio.to_json(fig1.to_dict()) + + +@pytest.mark.parametrize("fig_type_spec,fig_type", [ + ('Figure', go.Figure), + (go.Figure, go.Figure), + ('FigureWidget', go.FigureWidget), + (go.FigureWidget, go.FigureWidget) +]) +def test_read_json_from_file_string(fig1, fig_type_spec, fig_type): + with tempfile.TemporaryDirectory() as dir_name: + + # Write json file + path = os.path.join(dir_name, 'fig1.json') + with open(path, 'w') as f: + f.write(pio.to_json(fig1)) + + # read json from file as string + fig1_loaded = pio.read_json(path, output_type=fig_type_spec) + + # Check return type + assert isinstance(fig1_loaded, fig_type) + + # Check loaded figure + assert pio.to_json(fig1_loaded) == pio.to_json(fig1.to_dict()) + + +# write_json +# ---------- +@pytest.mark.parametrize('pretty', [True, False]) +@pytest.mark.parametrize('remove_uids', [True, False]) +def test_write_json_filelike(fig1, pretty, remove_uids): + # Configure file-like mock + filemock = MagicMock() + + # write_json to mock file + pio.write_json(fig1, filemock, pretty=pretty, remove_uids=remove_uids) + + # check write contents + expected = pio.to_json(fig1, pretty=pretty, remove_uids=remove_uids) + filemock.write.assert_called_once_with(expected) + + +@pytest.mark.parametrize('pretty', [True, False]) +@pytest.mark.parametrize('remove_uids', [True, False]) +def test_write_json_from_file_string(fig1, pretty, remove_uids): + with tempfile.TemporaryDirectory() as dir_name: + + # Write json + path = os.path.join(dir_name, 'fig1.json') + pio.write_json(fig1, path, pretty=pretty, remove_uids=remove_uids) + + # Open as text file + with open(path, 'r') as f: + result = f.read() + + # Check contents that were written + expected = pio.to_json(fig1, pretty=pretty, remove_uids=remove_uids) + assert result == expected diff --git a/tox.ini b/tox.ini index 25d10cc254..5c287d16b0 100644 --- a/tox.ini +++ b/tox.ini @@ -119,6 +119,7 @@ commands= nosetests {posargs} -x plotly/tests/test_core nosetests {posargs} -x plotly/tests/test_optional pytest _plotly_utils/tests/ + pytest plotly/tests/test_io [testenv:py34-optional] basepython={env:PLOTLY_TOX_PYTHON_34:} @@ -127,6 +128,7 @@ commands= nosetests {posargs} -x plotly/tests/test_core nosetests {posargs} -x plotly/tests/test_optional pytest _plotly_utils/tests/ + pytest plotly/tests/test_io [testenv:py35-optional] basepython={env:PLOTLY_TOX_PYTHON_35:} @@ -135,6 +137,7 @@ commands= nosetests {posargs} -x plotly/tests/test_core nosetests {posargs} -x plotly/tests/test_optional pytest _plotly_utils/tests/ + pytest plotly/tests/test_io [testenv:py36-optional] basepython={env:PLOTLY_TOX_PYTHON_36:} @@ -143,6 +146,7 @@ commands= nosetests {posargs} -x plotly/tests/test_core nosetests {posargs} -x plotly/tests/test_optional pytest _plotly_utils/tests/ + pytest plotly/tests/test_io [testenv:py37-optional] basepython={env:PLOTLY_TOX_PYTHON_37:} @@ -151,6 +155,7 @@ commands= nosetests {posargs} -x plotly/tests/test_core nosetests {posargs} -x plotly/tests/test_optional pytest _plotly_utils/tests/ + pytest plotly/tests/test_io ; Plot.ly environments [testenv:py27-plot_ly] From b6fc5aeb0515cb44eb84984bffe4566b98228d46 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Thu, 20 Sep 2018 07:37:29 -0400 Subject: [PATCH 2/6] Add ipywidgets to optional dependencies so that FigureWidget can be instantiated --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 5c287d16b0..c1af141152 100644 --- a/tox.ini +++ b/tox.ini @@ -61,6 +61,7 @@ deps= pytest==3.5.1 optional: numpy==1.11.3 optional: ipython[all]==5.1.0 + optional: ipywidgets==7.2.0 optional: ipykernel==4.8.2 optional: jupyter==1.0.0 optional: pandas==0.19.2 From 97f4de426e452ebcadcf629334f47764eaea001f Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Thu, 20 Sep 2018 07:52:40 -0400 Subject: [PATCH 3/6] pin matplotlib to version 2.2.3 --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index c1af141152..6375831ca8 100644 --- a/tox.ini +++ b/tox.ini @@ -70,6 +70,7 @@ deps= optional: geopandas==0.3.0 optional: pyshp==1.2.10 optional: pillow==5.2.0 + optional: matplotlib==2.2.3 ; CORE ENVIRONMENTS [testenv:py27-core] From d616f3a374fffcc90b8906a1db72f45c316455b3 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Thu, 20 Sep 2018 07:53:20 -0400 Subject: [PATCH 4/6] Don't assume FigureWidget is available in plotly.io._utils --- plotly/io/_utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/plotly/io/_utils.py b/plotly/io/_utils.py index 9b6ed825a3..8ce4989b8b 100644 --- a/plotly/io/_utils.py +++ b/plotly/io/_utils.py @@ -1,6 +1,6 @@ import plotly from plotly.basedatatypes import BaseFigure -from plotly.graph_objs import Figure, FigureWidget +import plotly.graph_objs as go def validate_coerce_fig_to_dict(fig, validate): @@ -20,10 +20,11 @@ def validate_coerce_fig_to_dict(fig, validate): def validate_coerce_output_type(output_type): - if output_type == 'Figure' or output_type == Figure: - cls = Figure - elif output_type == 'FigureWidget' or output_type == FigureWidget: - cls = FigureWidget + if output_type == 'Figure' or output_type == go.Figure: + cls = go.Figure + elif (output_type == 'FigureWidget' or + (hasattr(go, 'FigureWidget') and output_type == go.FigureWidget)): + cls = go.FigureWidget else: raise ValueError(""" Invalid output type: {output_type} From d84a4dd26c21961e8fa76bf9c6a2f69db0dc8f5d Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Thu, 20 Sep 2018 08:15:31 -0400 Subject: [PATCH 5/6] Add backports.tempfile for Python 2.7 testing compatibility --- optional-requirements.txt | 1 + plotly/tests/test_io/test_to_from_json.py | 3 ++- tox.ini | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/optional-requirements.txt b/optional-requirements.txt index 46362f78fd..f4a13ce86e 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -16,6 +16,7 @@ coverage==4.3.1 mock==2.0.0 nose==1.3.3 pytest==3.5.1 +backports.tempfile==1.0 ## orca ## psutil diff --git a/plotly/tests/test_io/test_to_from_json.py b/plotly/tests/test_io/test_to_from_json.py index 10c7991427..3566afd25c 100644 --- a/plotly/tests/test_io/test_to_from_json.py +++ b/plotly/tests/test_io/test_to_from_json.py @@ -4,13 +4,14 @@ import plotly import json import sys -import tempfile import os if sys.version_info.major == 3 and sys.version_info.minor >= 3: from unittest.mock import MagicMock + import tempfile else: from mock import MagicMock + from backports import tempfile # fixtures # -------- diff --git a/tox.ini b/tox.ini index 6375831ca8..aaa92637cc 100644 --- a/tox.ini +++ b/tox.ini @@ -59,6 +59,7 @@ deps= pytz==2016.10 retrying==1.3.3 pytest==3.5.1 + backports.tempfile==1.0 optional: numpy==1.11.3 optional: ipython[all]==5.1.0 optional: ipywidgets==7.2.0 From 44738d165cedd62c5a4deb36e6bf54b1b21a7453 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Fri, 21 Sep 2018 07:54:35 -0400 Subject: [PATCH 6/6] Review formatting updates --- plotly/io/_json.py | 13 +++++++++---- plotly/tests/test_io/test_to_from_json.py | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/plotly/io/_json.py b/plotly/io/_json.py index 0c4bda82ca..f65ecf0d07 100644 --- a/plotly/io/_json.py +++ b/plotly/io/_json.py @@ -109,12 +109,15 @@ def from_json(value, output_type='Figure', skip_invalid=False): ---------- value: str String containing the JSON representation of a figure + output_type: type or str (default 'Figure') The output figure type or type name. - One of: graph_objs.Figure, 'Figure, + One of: graph_objs.Figure, 'Figure', graph_objs.FigureWidget, 'FigureWidget' + skip_invalid: bool (default False) - False if invalid figure properties should result in an exception + False if invalid figure properties should result in an exception. + True if invalid figure properties should be silently ignored. Raises ------ @@ -162,10 +165,12 @@ def read_json(file, output_type='Figure', skip_invalid=False): output_type: type or str (default 'Figure') The output figure type or type name. - One of: graph_objs.Figure, 'Figure, + One of: graph_objs.Figure, 'Figure', graph_objs.FigureWidget, 'FigureWidget' + skip_invalid: bool (default False) - False if invalid figure properties should result in an exception + False if invalid figure properties should result in an exception. + True if invalid figure properties should be silently ignored. Returns ------- diff --git a/plotly/tests/test_io/test_to_from_json.py b/plotly/tests/test_io/test_to_from_json.py index 3566afd25c..82c720739b 100644 --- a/plotly/tests/test_io/test_to_from_json.py +++ b/plotly/tests/test_io/test_to_from_json.py @@ -13,6 +13,7 @@ from mock import MagicMock from backports import tempfile + # fixtures # -------- @pytest.fixture