From 40b9af19edc60e2d5b1eb5630f321938bbb21dee Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 5 Dec 2020 12:02:33 -0500
Subject: [PATCH 01/43] WIP accelerated encoding with orjson

---
 packages/python/plotly/_plotly_utils/utils.py |   2 +
 .../python/plotly/plotly/basedatatypes.py     |  29 ++-
 packages/python/plotly/plotly/io/_json.py     | 180 +++++++++++++++++-
 3 files changed, 193 insertions(+), 18 deletions(-)

diff --git a/packages/python/plotly/_plotly_utils/utils.py b/packages/python/plotly/_plotly_utils/utils.py
index 00351e1c7cf..40052d73e12 100644
--- a/packages/python/plotly/_plotly_utils/utils.py
+++ b/packages/python/plotly/_plotly_utils/utils.py
@@ -61,8 +61,10 @@ def encode(self, o):
         # We catch false positive cases (e.g. strings such as titles, labels etc.)
         # but this is ok since the intention is to skip the decoding / reencoding
         # step when it's completely safe
+
         if not ("NaN" in encoded_o or "Infinity" in encoded_o):
             return encoded_o
+
         # now:
         #    1. `loads` to switch Infinity, -Infinity, NaN to None
         #    2. `dumps` again so you get 'null' instead of extended JSON
diff --git a/packages/python/plotly/plotly/basedatatypes.py b/packages/python/plotly/plotly/basedatatypes.py
index 7954dea37f6..ab48c96d417 100644
--- a/packages/python/plotly/plotly/basedatatypes.py
+++ b/packages/python/plotly/plotly/basedatatypes.py
@@ -3273,7 +3273,7 @@ def _perform_batch_animate(self, animation_opts):
 
     # Exports
     # -------
-    def to_dict(self):
+    def to_dict(self, clone=True):
         """
         Convert figure to a dictionary
 
@@ -3286,23 +3286,33 @@ def to_dict(self):
         """
         # Handle data
         # -----------
-        data = deepcopy(self._data)
+        if clone:
+            data = deepcopy(self._data)
+        else:
+            data = self._data
 
         # Handle layout
         # -------------
-        layout = deepcopy(self._layout)
+        if clone:
+            layout = deepcopy(self._layout)
+        else:
+            layout = self._layout
 
         # Handle frames
         # -------------
         # Frame key is only added if there are any frames
         res = {"data": data, "layout": layout}
-        frames = deepcopy([frame._props for frame in self._frame_objs])
+        if clone:
+            frames = deepcopy([frame._props for frame in self._frame_objs])
+        else:
+            frames = [frame._props for frame in self._frame_objs]
+
         if frames:
             res["frames"] = frames
 
         return res
 
-    def to_plotly_json(self):
+    def to_plotly_json(self, clone=True):
         """
         Convert figure to a JSON representation as a Python dict
 
@@ -3310,7 +3320,7 @@ def to_plotly_json(self):
         -------
         dict
         """
-        return self.to_dict()
+        return self.to_dict(clone=clone)
 
     @staticmethod
     def _to_ordered_dict(d, skip_uid=False):
@@ -5524,7 +5534,7 @@ def on_change(self, callback, *args, **kwargs):
         # -----------------
         self._change_callbacks[arg_tuples].append(callback)
 
-    def to_plotly_json(self):
+    def to_plotly_json(self, clone=False):
         """
         Return plotly JSON representation of object as a Python dict
 
@@ -5532,7 +5542,10 @@ def to_plotly_json(self):
         -------
         dict
         """
-        return deepcopy(self._props if self._props is not None else {})
+        if clone:
+            return deepcopy(self._props if self._props is not None else {})
+        else:
+            return self._props if self._props is not None else {}
 
     @staticmethod
     def _vals_equal(v1, v2):
diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index f67dbab3eb6..84688a6c041 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -2,12 +2,28 @@
 
 from six import string_types
 import json
+import decimal
 
 
 from plotly.io._utils import validate_coerce_fig_to_dict, validate_coerce_output_type
+from _plotly_utils.utils import iso_to_plotly_time_string
+from _plotly_utils.optional_imports import get_module
+from _plotly_utils.basevalidators import ImageUriValidator
 
 
-def to_json(fig, validate=True, pretty=False, remove_uids=True):
+def coerce_to_strict(const):
+    """
+    This is used to ultimately *encode* into strict JSON, see `encode`
+
+    """
+    # before python 2.7, 'true', 'false', 'null', were include here.
+    if const in ("Infinity", "-Infinity", "NaN"):
+        return None
+    else:
+        return const
+
+
+def to_json(fig, validate=True, pretty=False, remove_uids=True, engine="auto"):
     """
     Convert a figure to a JSON string representation
 
@@ -32,7 +48,7 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True):
     str
         Representation of figure as a JSON string
     """
-    from _plotly_utils.utils import PlotlyJSONEncoder
+    orjson = get_module("orjson", should_load=True)
 
     # Validate figure
     # ---------------
@@ -44,16 +60,77 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True):
         for trace in fig_dict.get("data", []):
             trace.pop("uid", None)
 
+    # Determine json engine
+    if engine == "auto":
+        if orjson is not None:
+            engine = "orjson"
+        else:
+            engine = "json"
+    elif engine not in ["orjson", "json", "legacy"]:
+        raise ValueError("Invalid json engine: %s" % engine)
+
+    modules = {"sage_all": get_module("sage.all", should_load=False),
+               "np": get_module("numpy", should_load=False),
+               "pd": get_module("pandas", should_load=False),
+               "image": get_module("PIL.Image", should_load=False)}
+
+    orjson = get_module("orjson", should_load=True)
+
     # Dump to a JSON string and return
     # --------------------------------
-    opts = {"sort_keys": True}
-    if pretty:
-        opts["indent"] = 2
-    else:
-        # Remove all whitespace
-        opts["separators"] = (",", ":")
-
-    return json.dumps(fig_dict, cls=PlotlyJSONEncoder, **opts)
+    if engine in ("json", "legacy"):
+        opts = {"sort_keys": True}
+        if pretty:
+            opts["indent"] = 2
+        else:
+            # Remove all whitespace
+            opts["separators"] = (",", ":")
+
+        if engine == "json":
+            cleaned = clean_to_json_compatible(
+                fig, numpy_allowed=False,
+                non_finite_allowed=False,
+                datetime_allowed=False,
+                modules=modules,
+            )
+            encoded_o = json.dumps(cleaned, **opts)
+
+            if not ("NaN" in encoded_o or "Infinity" in encoded_o):
+                return encoded_o
+
+            # now:
+            #    1. `loads` to switch Infinity, -Infinity, NaN to None
+            #    2. `dumps` again so you get 'null' instead of extended JSON
+            try:
+                new_o = json.loads(encoded_o, parse_constant=coerce_to_strict)
+            except ValueError:
+
+                # invalid separators will fail here. raise a helpful exception
+                raise ValueError(
+                    "Encoding into strict JSON failed. Did you set the separators "
+                    "valid JSON separators?"
+                )
+            else:
+                return json.dumps(new_o, **opts)
+        else:
+            from _plotly_utils.utils import PlotlyJSONEncoder
+            return json.dumps(fig_dict, cls=PlotlyJSONEncoder, **opts)
+    elif engine == "orjson":
+        opts = (orjson.OPT_SORT_KEYS
+                | orjson.OPT_SERIALIZE_NUMPY
+                | orjson.OPT_OMIT_MICROSECONDS
+                )
+
+        if pretty:
+            opts |= orjson.OPT_INDENT_2
+
+        cleaned = clean_to_json_compatible(
+            fig, numpy_allowed=True,
+            non_finite_allowed=True,
+            datetime_allowed=True,
+            modules=modules,
+        )
+        return orjson.dumps(cleaned, option=opts).decode("utf8")
 
 
 def write_json(fig, file, validate=True, pretty=False, remove_uids=True):
@@ -194,3 +271,86 @@ def read_json(file, output_type="Figure", skip_invalid=False):
     # Construct and return figure
     # ---------------------------
     return from_json(json_str, skip_invalid=skip_invalid, output_type=output_type)
+
+
+def clean_to_json_compatible(obj, **kwargs):
+    # Try handling value as a scalar value that we have a conversion for.
+    # Return immediately if we know we've hit a primitive value
+
+    # unpack kwargs
+    numpy_allowed = kwargs.get("numpy_allowed", False)
+    non_finite_allowed = kwargs.get("non_finite_allowed", False)
+    datetime_allowed = kwargs.get("datetime_allowed", False)
+
+    modules = kwargs.get("modules", {})
+    sage_all = modules["sage_all"]
+    np = modules["np"]
+    pd = modules["pd"]
+    image = modules["image"]
+
+    # Plotly
+    try:
+        obj = obj.to_plotly_json(clone=False)
+    except (TypeError, NameError, ValueError):
+        # Try without clone for backward compatibility
+        obj = obj.to_plotly_json()
+    except AttributeError:
+        pass
+
+    # Sage
+    if sage_all is not None:
+        if obj in sage_all.RR:
+            return float(obj)
+        elif obj in sage_all.ZZ:
+            return int(obj)
+
+    # numpy
+    if np is not None:
+        if obj is np.ma.core.masked:
+            return float("nan")
+        elif numpy_allowed and isinstance(obj, np.ndarray) and obj.dtype.kind in ("b", "i", "u", "f"):
+            return obj
+
+    # pandas
+    if pd is not None:
+        if obj is pd.NaT:
+            return None
+        elif isinstance(obj, pd.Series):
+            if numpy_allowed and obj.dtype.kind in ("b", "i", "u", "f"):
+                return obj.values
+            elif datetime_allowed and obj.dtype.kind == "M":
+                return obj.dt.to_pydatetime().tolist()
+
+
+    # datetime and date
+    if not datetime_allowed:
+        try:
+            # Is this cleanup still needed?
+            return iso_to_plotly_time_string(obj.isoformat())
+        except AttributeError:
+            pass
+
+    # Try .tolist() convertible
+    try:
+        # obj = obj.tolist()
+        return obj.tolist()
+    except AttributeError:
+        pass
+
+    # Do best we can with decimal
+    if isinstance(obj, decimal.Decimal):
+        return float(obj)
+
+    # PIL
+    if image is not None and isinstance(obj, image.Image):
+        return ImageUriValidator.pil_image_to_uri(obj)
+
+    # Recurse into lists and dictionaries
+    if isinstance(obj, dict):
+        return {k: clean_to_json_compatible(v, **kwargs) for k, v in obj.items()}
+    elif isinstance(obj, (list, tuple)):
+        if obj:
+            # Must process list recursively even though it may be slow
+            return [clean_to_json_compatible(v, **kwargs) for v in obj]
+
+    return obj

From f79e318a5618962ec13bd7cd6680ed0d0523d2b5 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 5 Dec 2020 13:25:51 -0500
Subject: [PATCH 02/43] support fig to dict in io without cloning

---
 packages/python/plotly/plotly/io/_json.py  | 6 +++---
 packages/python/plotly/plotly/io/_utils.py | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 84688a6c041..141244a01a4 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -52,7 +52,7 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine="auto"):
 
     # Validate figure
     # ---------------
-    fig_dict = validate_coerce_fig_to_dict(fig, validate)
+    fig_dict = validate_coerce_fig_to_dict(fig, validate, clone=False)
 
     # Remove trace uid
     # ----------------
@@ -88,7 +88,7 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine="auto"):
 
         if engine == "json":
             cleaned = clean_to_json_compatible(
-                fig, numpy_allowed=False,
+                fig_dict, numpy_allowed=False,
                 non_finite_allowed=False,
                 datetime_allowed=False,
                 modules=modules,
@@ -125,7 +125,7 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine="auto"):
             opts |= orjson.OPT_INDENT_2
 
         cleaned = clean_to_json_compatible(
-            fig, numpy_allowed=True,
+            fig_dict, numpy_allowed=True,
             non_finite_allowed=True,
             datetime_allowed=True,
             modules=modules,
diff --git a/packages/python/plotly/plotly/io/_utils.py b/packages/python/plotly/plotly/io/_utils.py
index b3b376e9d89..000cb56b01d 100644
--- a/packages/python/plotly/plotly/io/_utils.py
+++ b/packages/python/plotly/plotly/io/_utils.py
@@ -4,11 +4,11 @@
 import plotly.graph_objs as go
 
 
-def validate_coerce_fig_to_dict(fig, validate):
+def validate_coerce_fig_to_dict(fig, validate, clone=True):
     from plotly.basedatatypes import BaseFigure
 
     if isinstance(fig, BaseFigure):
-        fig_dict = fig.to_dict()
+        fig_dict = fig.to_dict(clone=clone)
     elif isinstance(fig, dict):
         if validate:
             # This will raise an exception if fig is not a valid plotly figure

From 7b3593a5f2fd658760649982524e5af1cd7cfa05 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 5 Dec 2020 13:55:24 -0500
Subject: [PATCH 03/43] fix clone default

---
 packages/python/plotly/plotly/basedatatypes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/python/plotly/plotly/basedatatypes.py b/packages/python/plotly/plotly/basedatatypes.py
index ab48c96d417..8315ad92e81 100644
--- a/packages/python/plotly/plotly/basedatatypes.py
+++ b/packages/python/plotly/plotly/basedatatypes.py
@@ -5534,7 +5534,7 @@ def on_change(self, callback, *args, **kwargs):
         # -----------------
         self._change_callbacks[arg_tuples].append(callback)
 
-    def to_plotly_json(self, clone=False):
+    def to_plotly_json(self, clone=True):
         """
         Return plotly JSON representation of object as a Python dict
 

From da915d6848fd5d0c9b6b59560894ef1406d0b0a5 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 5 Dec 2020 14:59:20 -0500
Subject: [PATCH 04/43] Add pio.json.config object to configure default encoder

Later we can use this to configure base64 encoding
---
 .../python/plotly/plotly/basedatatypes.py     | 16 ++++++
 packages/python/plotly/plotly/io/__init__.py  |  4 +-
 packages/python/plotly/plotly/io/_json.py     | 57 ++++++++++++++++++-
 packages/python/plotly/plotly/io/json.py      |  1 +
 4 files changed, 74 insertions(+), 4 deletions(-)
 create mode 100644 packages/python/plotly/plotly/io/json.py

diff --git a/packages/python/plotly/plotly/basedatatypes.py b/packages/python/plotly/plotly/basedatatypes.py
index 8315ad92e81..8bfba5e9486 100644
--- a/packages/python/plotly/plotly/basedatatypes.py
+++ b/packages/python/plotly/plotly/basedatatypes.py
@@ -3423,6 +3423,14 @@ def to_json(self, *args, **kwargs):
         remove_uids: bool (default True)
             True if trace UIDs should be omitted from the JSON representation
 
+        engine: str (default None)
+            The JSON encoding engine to use. One of:
+              - "json" for a rewritten encoder based on the built-in Python json module
+              - "orjson" for a fast encoder the requires the orjson package
+              - "legacy" for the legacy JSON encoder.
+            If not specified, the default encoder is set to the current value of
+            plotly.io.json.config.default_encoder.
+
         Returns
         -------
         str
@@ -3479,6 +3487,14 @@ def write_json(self, *args, **kwargs):
         remove_uids: bool (default True)
             True if trace UIDs should be omitted from the JSON representation
 
+        engine: str (default None)
+            The JSON encoding engine to use. One of:
+              - "json" for a rewritten encoder based on the built-in Python json module
+              - "orjson" for a fast encoder the requires the orjson package
+              - "legacy" for the legacy JSON encoder.
+            If not specified, the default encoder is set to the current value of
+            plotly.io.json.config.default_encoder.
+
         Returns
         -------
         None
diff --git a/packages/python/plotly/plotly/io/__init__.py b/packages/python/plotly/plotly/io/__init__.py
index e1d1e5be8d7..8c53557251e 100644
--- a/packages/python/plotly/plotly/io/__init__.py
+++ b/packages/python/plotly/plotly/io/__init__.py
@@ -4,6 +4,7 @@
 if sys.version_info < (3, 7):
     from ._kaleido import to_image, write_image, full_figure_for_development
     from . import orca, kaleido
+    from . import json
     from ._json import to_json, from_json, read_json, write_json
     from ._templates import templates, to_templated
     from ._html import to_html, write_html
@@ -14,6 +15,7 @@
         "to_image",
         "write_image",
         "orca",
+        "json",
         "to_json",
         "from_json",
         "read_json",
@@ -30,7 +32,7 @@
 else:
     __all__, __getattr__, __dir__ = relative_import(
         __name__,
-        [".orca", ".kaleido", ".base_renderers"],
+        [".orca", ".kaleido", ".json", ".base_renderers"],
         [
             "._kaleido.to_image",
             "._kaleido.write_image",
diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 141244a01a4..8b89f6ecf69 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -11,6 +11,39 @@
 from _plotly_utils.basevalidators import ImageUriValidator
 
 
+# Orca configuration class
+# ------------------------
+class JsonConfig(object):
+    _valid_encoders = ("legacy", "json", "orjson", "auto")
+
+    def __init__(self):
+        self._default_encoder = "auto"
+
+    @property
+    def default_encoder(self):
+        return self._default_encoder
+
+    @default_encoder.setter
+    def default_encoder(self, val):
+        if val not in JsonConfig._valid_encoders:
+            raise ValueError(
+                "Supported JSON encoders include {valid}\n"
+                "    Received {val}".format(valid=JsonConfig._valid_encoders, val=val)
+            )
+
+        if val == "orjson":
+            orjson = get_module("orjson")
+            if orjson is None:
+                raise ValueError(
+                    "The orjson encoder requires the orjson package"
+                )
+
+        self._default_encoder = val
+
+
+config = JsonConfig()
+
+
 def coerce_to_strict(const):
     """
     This is used to ultimately *encode* into strict JSON, see `encode`
@@ -23,7 +56,7 @@ def coerce_to_strict(const):
         return const
 
 
-def to_json(fig, validate=True, pretty=False, remove_uids=True, engine="auto"):
+def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
     """
     Convert a figure to a JSON string representation
 
@@ -43,6 +76,14 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine="auto"):
     remove_uids: bool (default True)
         True if trace UIDs should be omitted from the JSON representation
 
+    engine: str (default None)
+        The JSON encoding engine to use. One of:
+          - "json" for a rewritten encoder based on the built-in Python json module
+          - "orjson" for a fast encoder the requires the orjson package
+          - "legacy" for the legacy JSON encoder.
+        If not specified, the default encoder is set to the current value of
+        plotly.io.json.config.default_encoder.
+
     Returns
     -------
     str
@@ -61,6 +102,9 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine="auto"):
             trace.pop("uid", None)
 
     # Determine json engine
+    if engine is None:
+        engine = config.default_encoder
+
     if engine == "auto":
         if orjson is not None:
             engine = "orjson"
@@ -133,7 +177,7 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine="auto"):
         return orjson.dumps(cleaned, option=opts).decode("utf8")
 
 
-def write_json(fig, file, validate=True, pretty=False, remove_uids=True):
+def write_json(fig, file, validate=True, pretty=False, remove_uids=True, engine=None):
     """
     Convert a figure to JSON and write it to a file or writeable
     object
@@ -154,6 +198,13 @@ def write_json(fig, file, validate=True, pretty=False, remove_uids=True):
     remove_uids: bool (default True)
         True if trace UIDs should be omitted from the JSON representation
 
+    engine: str (default None)
+        The JSON encoding engine to use. One of:
+          - "json" for a rewritten encoder based on the built-in Python json module
+          - "orjson" for a fast encoder the requires the orjson package
+          - "legacy" for the legacy JSON encoder.
+        If not specified, the default encoder is set to the current value of
+        plotly.io.json.config.default_encoder.
     Returns
     -------
     None
@@ -162,7 +213,7 @@ def write_json(fig, file, validate=True, pretty=False, remove_uids=True):
     # Get JSON string
     # ---------------
     # Pass through validate argument and let to_json handle validation logic
-    json_str = to_json(fig, validate=validate, pretty=pretty, remove_uids=remove_uids)
+    json_str = to_json(fig, validate=validate, pretty=pretty, remove_uids=remove_uids, engine=engine)
 
     # Check if file is a string
     # -------------------------
diff --git a/packages/python/plotly/plotly/io/json.py b/packages/python/plotly/plotly/io/json.py
new file mode 100644
index 00000000000..009fae4f4ec
--- /dev/null
+++ b/packages/python/plotly/plotly/io/json.py
@@ -0,0 +1 @@
+from ._json import to_json, write_json, from_json, read_json, config

From 7b235ef43eaf17069bfac4765604f4066e977784 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 5 Dec 2020 15:07:11 -0500
Subject: [PATCH 05/43] default_encoder to default_engine

---
 packages/python/plotly/plotly/io/_json.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 8b89f6ecf69..c93498e8106 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -17,14 +17,14 @@ class JsonConfig(object):
     _valid_encoders = ("legacy", "json", "orjson", "auto")
 
     def __init__(self):
-        self._default_encoder = "auto"
+        self._default_engine = "auto"
 
     @property
-    def default_encoder(self):
-        return self._default_encoder
+    def default_engine(self):
+        return self._default_engine
 
-    @default_encoder.setter
-    def default_encoder(self, val):
+    @default_engine.setter
+    def default_engine(self, val):
         if val not in JsonConfig._valid_encoders:
             raise ValueError(
                 "Supported JSON encoders include {valid}\n"
@@ -38,7 +38,7 @@ def default_encoder(self, val):
                     "The orjson encoder requires the orjson package"
                 )
 
-        self._default_encoder = val
+        self._default_engine = val
 
 
 config = JsonConfig()
@@ -103,7 +103,7 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
 
     # Determine json engine
     if engine is None:
-        engine = config.default_encoder
+        engine = config.default_engine
 
     if engine == "auto":
         if orjson is not None:

From 7895b6a0f72b9ebfe864ffaa960daec4be9ead0d Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 5 Dec 2020 15:25:20 -0500
Subject: [PATCH 06/43] blacken

---
 packages/python/plotly/plotly/io/_json.py | 41 ++++++++++++++---------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index c93498e8106..bf4720a3d17 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -34,9 +34,7 @@ def default_engine(self, val):
         if val == "orjson":
             orjson = get_module("orjson")
             if orjson is None:
-                raise ValueError(
-                    "The orjson encoder requires the orjson package"
-                )
+                raise ValueError("The orjson encoder requires the orjson package")
 
         self._default_engine = val
 
@@ -113,10 +111,12 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
     elif engine not in ["orjson", "json", "legacy"]:
         raise ValueError("Invalid json engine: %s" % engine)
 
-    modules = {"sage_all": get_module("sage.all", should_load=False),
-               "np": get_module("numpy", should_load=False),
-               "pd": get_module("pandas", should_load=False),
-               "image": get_module("PIL.Image", should_load=False)}
+    modules = {
+        "sage_all": get_module("sage.all", should_load=False),
+        "np": get_module("numpy", should_load=False),
+        "pd": get_module("pandas", should_load=False),
+        "image": get_module("PIL.Image", should_load=False),
+    }
 
     orjson = get_module("orjson", should_load=True)
 
@@ -132,7 +132,8 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
 
         if engine == "json":
             cleaned = clean_to_json_compatible(
-                fig_dict, numpy_allowed=False,
+                fig_dict,
+                numpy_allowed=False,
                 non_finite_allowed=False,
                 datetime_allowed=False,
                 modules=modules,
@@ -158,18 +159,21 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
                 return json.dumps(new_o, **opts)
         else:
             from _plotly_utils.utils import PlotlyJSONEncoder
+
             return json.dumps(fig_dict, cls=PlotlyJSONEncoder, **opts)
     elif engine == "orjson":
-        opts = (orjson.OPT_SORT_KEYS
-                | orjson.OPT_SERIALIZE_NUMPY
-                | orjson.OPT_OMIT_MICROSECONDS
-                )
+        opts = (
+            orjson.OPT_SORT_KEYS
+            | orjson.OPT_SERIALIZE_NUMPY
+            | orjson.OPT_OMIT_MICROSECONDS
+        )
 
         if pretty:
             opts |= orjson.OPT_INDENT_2
 
         cleaned = clean_to_json_compatible(
-            fig_dict, numpy_allowed=True,
+            fig_dict,
+            numpy_allowed=True,
             non_finite_allowed=True,
             datetime_allowed=True,
             modules=modules,
@@ -213,7 +217,9 @@ def write_json(fig, file, validate=True, pretty=False, remove_uids=True, engine=
     # Get JSON string
     # ---------------
     # Pass through validate argument and let to_json handle validation logic
-    json_str = to_json(fig, validate=validate, pretty=pretty, remove_uids=remove_uids, engine=engine)
+    json_str = to_json(
+        fig, validate=validate, pretty=pretty, remove_uids=remove_uids, engine=engine
+    )
 
     # Check if file is a string
     # -------------------------
@@ -359,7 +365,11 @@ def clean_to_json_compatible(obj, **kwargs):
     if np is not None:
         if obj is np.ma.core.masked:
             return float("nan")
-        elif numpy_allowed and isinstance(obj, np.ndarray) and obj.dtype.kind in ("b", "i", "u", "f"):
+        elif (
+            numpy_allowed
+            and isinstance(obj, np.ndarray)
+            and obj.dtype.kind in ("b", "i", "u", "f")
+        ):
             return obj
 
     # pandas
@@ -372,7 +382,6 @@ def clean_to_json_compatible(obj, **kwargs):
             elif datetime_allowed and obj.dtype.kind == "M":
                 return obj.dt.to_pydatetime().tolist()
 
-
     # datetime and date
     if not datetime_allowed:
         try:

From ce05a68965a252a1756d6eac64bf319ef17ed158 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sun, 6 Dec 2020 13:25:51 -0500
Subject: [PATCH 07/43] Handle Dash objects in to_json

---
 packages/python/plotly/plotly/io/_utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/python/plotly/plotly/io/_utils.py b/packages/python/plotly/plotly/io/_utils.py
index 000cb56b01d..6b5f8b40f81 100644
--- a/packages/python/plotly/plotly/io/_utils.py
+++ b/packages/python/plotly/plotly/io/_utils.py
@@ -15,6 +15,8 @@ def validate_coerce_fig_to_dict(fig, validate, clone=True):
             fig_dict = plotly.graph_objs.Figure(fig).to_plotly_json()
         else:
             fig_dict = fig
+    elif hasattr(fig, "to_plotly_json"):
+        fig_dict = fig.to_plotly_json()
     else:
         raise ValueError(
             """

From 4ef651054c5ff43a2d89b1ae975ce0f46dc20ab5 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 11:13:02 -0500
Subject: [PATCH 08/43] add JSON encoding tests

---
 packages/python/plotly/plotly/io/_json.py     | 277 +++++++++++++-----
 packages/python/plotly/plotly/io/json.py      |  10 +-
 .../tests/test_io/test_to_from_plotly_json.py | 149 ++++++++++
 3 files changed, 359 insertions(+), 77 deletions(-)
 create mode 100644 packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index bf4720a3d17..d809e702cc9 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -3,10 +3,10 @@
 from six import string_types
 import json
 import decimal
+import os
 
 
 from plotly.io._utils import validate_coerce_fig_to_dict, validate_coerce_output_type
-from _plotly_utils.utils import iso_to_plotly_time_string
 from _plotly_utils.optional_imports import get_module
 from _plotly_utils.basevalidators import ImageUriValidator
 
@@ -14,10 +14,10 @@
 # Orca configuration class
 # ------------------------
 class JsonConfig(object):
-    _valid_encoders = ("legacy", "json", "orjson", "auto")
+    _valid_engines = ("legacy", "json", "orjson", "auto")
 
     def __init__(self):
-        self._default_engine = "auto"
+        self._default_engine = "legacy"
 
     @property
     def default_engine(self):
@@ -25,16 +25,16 @@ def default_engine(self):
 
     @default_engine.setter
     def default_engine(self, val):
-        if val not in JsonConfig._valid_encoders:
+        if val not in JsonConfig._valid_engines:
             raise ValueError(
-                "Supported JSON encoders include {valid}\n"
-                "    Received {val}".format(valid=JsonConfig._valid_encoders, val=val)
+                "Supported JSON engines include {valid}\n"
+                "    Received {val}".format(valid=JsonConfig._valid_engines, val=val)
             )
 
         if val == "orjson":
             orjson = get_module("orjson")
             if orjson is None:
-                raise ValueError("The orjson encoder requires the orjson package")
+                raise ValueError("The orjson engine requires the orjson package")
 
         self._default_engine = val
 
@@ -54,51 +54,39 @@ def coerce_to_strict(const):
         return const
 
 
-def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
+def to_plotly_json(plotly_object, pretty=False, engine=None):
     """
-    Convert a figure to a JSON string representation
+    Convert a plotly/Dash object to a JSON string representation
 
     Parameters
     ----------
-    fig:
-        Figure object or dict representing a figure
-
-    validate: bool (default True)
-        True if the figure should be validated before being converted to
-        JSON, False otherwise.
+    plotly_object:
+        A plotly/Dash object represented as a dict, graph_object, or Dash component
 
     pretty: bool (default False)
         True if JSON representation should be pretty-printed, False if
         representation should be as compact as possible.
 
-    remove_uids: bool (default True)
-        True if trace UIDs should be omitted from the JSON representation
-
     engine: str (default None)
         The JSON encoding engine to use. One of:
-          - "json" for a rewritten encoder based on the built-in Python json module
-          - "orjson" for a fast encoder the requires the orjson package
-          - "legacy" for the legacy JSON encoder.
-        If not specified, the default encoder is set to the current value of
-        plotly.io.json.config.default_encoder.
+          - "json" for an engine based on the built-in Python json module
+          - "orjson" for a faster engine that requires the orjson package
+          - "legacy" for the legacy JSON engine.
+          - "auto" for the "orjson" engine if available, otherwise "json"
+        If not specified, the default engine is set to the current value of
+        plotly.io.json.config.default_engine.
 
     Returns
     -------
     str
-        Representation of figure as a JSON string
+        Representation of input object as a JSON string
+
+    See Also
+    --------
+    to_json : Convert a plotly Figure to JSON with validation
     """
     orjson = get_module("orjson", should_load=True)
 
-    # Validate figure
-    # ---------------
-    fig_dict = validate_coerce_fig_to_dict(fig, validate, clone=False)
-
-    # Remove trace uid
-    # ----------------
-    if remove_uids:
-        for trace in fig_dict.get("data", []):
-            trace.pop("uid", None)
-
     # Determine json engine
     if engine is None:
         engine = config.default_engine
@@ -132,9 +120,8 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
 
         if engine == "json":
             cleaned = clean_to_json_compatible(
-                fig_dict,
+                plotly_object,
                 numpy_allowed=False,
-                non_finite_allowed=False,
                 datetime_allowed=False,
                 modules=modules,
             )
@@ -149,7 +136,6 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
             try:
                 new_o = json.loads(encoded_o, parse_constant=coerce_to_strict)
             except ValueError:
-
                 # invalid separators will fail here. raise a helpful exception
                 raise ValueError(
                     "Encoding into strict JSON failed. Did you set the separators "
@@ -160,27 +146,70 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
         else:
             from _plotly_utils.utils import PlotlyJSONEncoder
 
-            return json.dumps(fig_dict, cls=PlotlyJSONEncoder, **opts)
+            return json.dumps(plotly_object, cls=PlotlyJSONEncoder, **opts)
     elif engine == "orjson":
-        opts = (
-            orjson.OPT_SORT_KEYS
-            | orjson.OPT_SERIALIZE_NUMPY
-            | orjson.OPT_OMIT_MICROSECONDS
-        )
+        opts = orjson.OPT_SORT_KEYS | orjson.OPT_SERIALIZE_NUMPY
 
         if pretty:
             opts |= orjson.OPT_INDENT_2
 
         cleaned = clean_to_json_compatible(
-            fig_dict,
-            numpy_allowed=True,
-            non_finite_allowed=True,
-            datetime_allowed=True,
-            modules=modules,
+            plotly_object, numpy_allowed=True, datetime_allowed=True, modules=modules,
         )
         return orjson.dumps(cleaned, option=opts).decode("utf8")
 
 
+def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
+    """
+    Convert a figure to a JSON string representation
+
+    Parameters
+    ----------
+    fig:
+        Figure object or dict representing a figure
+
+    validate: bool (default True)
+        True if the figure should be validated before being converted to
+        JSON, False otherwise.
+
+    pretty: bool (default False)
+        True if JSON representation should be pretty-printed, False if
+        representation should be as compact as possible.
+
+    remove_uids: bool (default True)
+        True if trace UIDs should be omitted from the JSON representation
+
+    engine: str (default None)
+        The JSON encoding engine to use. One of:
+          - "json" for an engine based on the built-in Python json module
+          - "orjson" for a faster engine that requires the orjson package
+          - "legacy" for the legacy JSON engine.
+          - "auto" for the "orjson" engine if available, otherwise "json"
+        If not specified, the default engine is set to the current value of
+        plotly.io.json.config.default_engine.
+
+    Returns
+    -------
+    str
+        Representation of figure as a JSON string
+
+    See Also
+    --------
+    to_plotly_json : Convert an arbitrary plotly graph_object or Dash component to JSON
+    """
+    # Validate figure
+    # ---------------
+    fig_dict = validate_coerce_fig_to_dict(fig, validate, clone=False)
+
+    # Remove trace uid
+    # ----------------
+    if remove_uids:
+        for trace in fig_dict.get("data", []):
+            trace.pop("uid", None)
+
+    return to_plotly_json(fig_dict, pretty=pretty, engine=engine)
+
+
 def write_json(fig, file, validate=True, pretty=False, remove_uids=True, engine=None):
     """
     Convert a figure to JSON and write it to a file or writeable
@@ -204,11 +233,12 @@ def write_json(fig, file, validate=True, pretty=False, remove_uids=True, engine=
 
     engine: str (default None)
         The JSON encoding engine to use. One of:
-          - "json" for a rewritten encoder based on the built-in Python json module
-          - "orjson" for a fast encoder the requires the orjson package
-          - "legacy" for the legacy JSON encoder.
-        If not specified, the default encoder is set to the current value of
-        plotly.io.json.config.default_encoder.
+          - "json" for an engine based on the built-in Python json module
+          - "orjson" for a faster engine that requires the orjson package
+          - "legacy" for the legacy JSON engine.
+          - "auto" for the "orjson" engine if available, otherwise "json"
+        If not specified, the default engine is set to the current value of
+        plotly.io.json.config.default_engine.
     Returns
     -------
     None
@@ -234,7 +264,67 @@ def write_json(fig, file, validate=True, pretty=False, remove_uids=True, engine=
         file.write(json_str)
 
 
-def from_json(value, output_type="Figure", skip_invalid=False):
+def from_plotly_json(value, engine=None):
+    """
+    Parse JSON string using the specified JSON engine
+
+    Parameters
+    ----------
+    value: str
+        A JSON string
+
+    engine: str (default None)
+        The JSON decoding engine to use. One of:
+          - if "json" or "legacy", parse JSON using built in json module
+          - if "orjson", parse using the faster orjson module, requires the orjson
+            package
+          - if "auto" use orjson module if available, otherwise use the json module
+
+        If not specified, the default engine is set to the current value of
+        plotly.io.json.config.default_engine.
+
+    Returns
+    -------
+    dict
+    """
+    # Validate value
+    # --------------
+    if not isinstance(value, (string_types, bytes)):
+        raise ValueError(
+            """
+from_plotly_json requires a string or bytes argument but received value of type {typ}
+    Received value: {value}""".format(
+                typ=type(value), value=value
+            )
+        )
+
+    orjson = get_module("orjson", should_load=True)
+
+    # Determine json engine
+    if engine is None:
+        engine = config.default_engine
+
+    if engine == "auto":
+        if orjson is not None:
+            engine = "orjson"
+        else:
+            engine = "json"
+    elif engine not in ["orjson", "json", "legacy"]:
+        raise ValueError("Invalid json engine: %s" % engine)
+
+    if engine == "orjson":
+        # orjson handles bytes input natively
+        value_dict = orjson.loads(value)
+    else:
+        # decode bytes to str for built-in json module
+        if isinstance(value, bytes):
+            value = value.decode("utf-8")
+        value_dict = json.loads(value)
+
+    return value_dict
+
+
+def from_json(value, output_type="Figure", skip_invalid=False, engine=None):
     """
     Construct a figure from a JSON string
 
@@ -251,6 +341,16 @@ def from_json(value, output_type="Figure", skip_invalid=False):
         False if invalid figure properties should result in an exception.
         True if invalid figure properties should be silently ignored.
 
+    engine: str (default None)
+        The JSON decoding engine to use. One of:
+          - if "json" or "legacy", parse JSON using built in json module
+          - if "orjson", parse using the faster orjson module, requires the orjson
+            package
+          - if "auto" use orjson module if available, otherwise use the json module
+
+        If not specified, the default engine is set to the current value of
+        plotly.io.json.config.default_engine.
+
     Raises
     ------
     ValueError
@@ -262,20 +362,9 @@ def from_json(value, output_type="Figure", skip_invalid=False):
     Figure or FigureWidget
     """
 
-    # Validate value
-    # --------------
-    if not isinstance(value, string_types):
-        raise ValueError(
-            """
-from_json requires a string argument but received value of type {typ}
-    Received value: {value}""".format(
-                typ=type(value), value=value
-            )
-        )
-
     # Decode JSON
     # -----------
-    fig_dict = json.loads(value)
+    fig_dict = from_plotly_json(value, engine=engine)
 
     # Validate coerce output type
     # ---------------------------
@@ -287,7 +376,7 @@ def from_json(value, output_type="Figure", skip_invalid=False):
     return fig
 
 
-def read_json(file, output_type="Figure", skip_invalid=False):
+def read_json(file, output_type="Figure", skip_invalid=False, engine=None):
     """
     Construct a figure from the JSON contents of a local file or readable
     Python object
@@ -306,6 +395,16 @@ def read_json(file, output_type="Figure", skip_invalid=False):
         False if invalid figure properties should result in an exception.
         True if invalid figure properties should be silently ignored.
 
+    engine: str (default None)
+        The JSON decoding engine to use. One of:
+          - if "json" or "legacy", parse JSON using built in json module
+          - if "orjson", parse using the faster orjson module, requires the orjson
+            package
+          - if "auto" use orjson module if available, otherwise use the json module
+
+        If not specified, the default engine is set to the current value of
+        plotly.io.json.config.default_engine.
+
     Returns
     -------
     Figure or FigureWidget
@@ -327,16 +426,21 @@ def read_json(file, output_type="Figure", skip_invalid=False):
 
     # Construct and return figure
     # ---------------------------
-    return from_json(json_str, skip_invalid=skip_invalid, output_type=output_type)
+    return from_json(
+        json_str, skip_invalid=skip_invalid, output_type=output_type, engine=engine
+    )
 
 
 def clean_to_json_compatible(obj, **kwargs):
     # Try handling value as a scalar value that we have a conversion for.
     # Return immediately if we know we've hit a primitive value
 
+    # Bail out fast for simple scalar types
+    if isinstance(obj, (int, float, string_types)):
+        return obj
+
     # unpack kwargs
     numpy_allowed = kwargs.get("numpy_allowed", False)
-    non_finite_allowed = kwargs.get("non_finite_allowed", False)
     datetime_allowed = kwargs.get("datetime_allowed", False)
 
     modules = kwargs.get("modules", {})
@@ -376,23 +480,44 @@ def clean_to_json_compatible(obj, **kwargs):
     if pd is not None:
         if obj is pd.NaT:
             return None
-        elif isinstance(obj, pd.Series):
+        elif isinstance(obj, (pd.Series, pd.DatetimeIndex)):
             if numpy_allowed and obj.dtype.kind in ("b", "i", "u", "f"):
                 return obj.values
-            elif datetime_allowed and obj.dtype.kind == "M":
-                return obj.dt.to_pydatetime().tolist()
+            elif obj.dtype.kind == "M":
+                if isinstance(obj, pd.Series):
+                    dt_values = obj.dt.to_pydatetime().tolist()
+                else:  # DatetimeIndex
+                    dt_values = obj.to_pydatetime().tolist()
+
+                if not datetime_allowed:
+                    # Note: We don't need to handle dropping timezones here because
+                    # numpy's datetime64 doesn't support them and pandas's tolist()
+                    # doesn't preserve them.
+                    for i in range(len(dt_values)):
+                        dt_values[i] = dt_values[i].isoformat()
+
+                return dt_values
 
     # datetime and date
     if not datetime_allowed:
         try:
-            # Is this cleanup still needed?
-            return iso_to_plotly_time_string(obj.isoformat())
+            # Need to drop timezone for scalar datetimes
+            return obj.replace(tzinfo=None).isoformat()
+        except (TypeError, AttributeError):
+            pass
+
+        if np and isinstance(obj, np.datetime64):
+            return str(obj)
+    else:
+        try:
+            # Need to drop timezone for scalar datetimes. Don't need to convert
+            # to string since engine can do that
+            return obj.replace(tzinfo=None)
         except AttributeError:
             pass
 
-    # Try .tolist() convertible
+    # Try .tolist() convertible, do not recurse inside
     try:
-        # obj = obj.tolist()
         return obj.tolist()
     except AttributeError:
         pass
diff --git a/packages/python/plotly/plotly/io/json.py b/packages/python/plotly/plotly/io/json.py
index 009fae4f4ec..8f895dc81a7 100644
--- a/packages/python/plotly/plotly/io/json.py
+++ b/packages/python/plotly/plotly/io/json.py
@@ -1 +1,9 @@
-from ._json import to_json, write_json, from_json, read_json, config
+from ._json import (
+    to_json,
+    write_json,
+    from_json,
+    read_json,
+    config,
+    to_plotly_json,
+    from_plotly_json,
+)
diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
new file mode 100644
index 00000000000..2cc571488fe
--- /dev/null
+++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
@@ -0,0 +1,149 @@
+import plotly.io.json as pio
+import plotly.graph_objects as go
+import numpy as np
+import pandas as pd
+import pytest
+import json
+import datetime
+from pytz import timezone
+
+eastern = timezone("US/Eastern")
+
+
+# Testing helper
+def build_json_opts(pretty=False):
+    opts = {"sort_keys": True}
+    if pretty:
+        opts["indent"] = 2
+    else:
+        opts["separators"] = (",", ":")
+    return opts
+
+
+def to_json_test(value, pretty=False):
+    return json.dumps(value, **build_json_opts(pretty=pretty))
+
+
+def isoformat_test(dt_value):
+    if isinstance(dt_value, np.datetime64):
+        return str(dt_value)
+    elif isinstance(dt_value, datetime.datetime):
+        return dt_value.replace(tzinfo=None).isoformat()
+    else:
+        raise ValueError("Unsupported date type: {}".format(type(dt_value)))
+
+
+def build_test_dict(value):
+    return dict(a=value, b=[3, value], c=dict(Z=value))
+
+
+def build_test_dict_string(value_string, pretty=False):
+    if pretty:
+        non_pretty_str = build_test_dict_string(value_string, pretty=False)
+        return to_json_test(json.loads(non_pretty_str), pretty=True)
+    else:
+        value_string = str(value_string).replace(" ", "")
+        return """{"a":%s,"b":[3,%s],"c":{"Z":%s}}""" % tuple([value_string] * 3)
+
+
+# Fixtures
+@pytest.fixture(scope="module", params=["json", "orjson", "legacy", "auto"])
+def engine(request):
+    return request.param
+
+
+@pytest.fixture(scope="module", params=[False])
+def pretty(request):
+    return request.param
+
+
+@pytest.fixture(scope="module", params=["float64", "int32", "uint32"])
+def graph_object(request):
+    return request.param
+
+
+@pytest.fixture(scope="module", params=["float64", "int32", "uint32"])
+def numeric_numpy_array(request):
+    dtype = request.param
+    return np.linspace(-5, 5, 4, dtype=dtype)
+
+
+@pytest.fixture(scope="module")
+def object_numpy_array(request):
+    return np.array(["a", 1, [2, 3]])
+
+
+@pytest.fixture(
+    scope="module",
+    params=[
+        datetime.datetime(2003, 7, 12, 8, 34, 22),
+        datetime.datetime.now(),
+        np.datetime64(datetime.datetime.utcnow()),
+        eastern.localize(datetime.datetime(2003, 7, 12, 8, 34, 22)),
+        eastern.localize(datetime.datetime.now()),
+    ],
+)
+def datetime_value(request):
+    return request.param
+
+
+@pytest.fixture(
+    params=[
+        lambda a: pd.DatetimeIndex(a),  # Pandas DatetimeIndex
+        lambda a: pd.Series(pd.DatetimeIndex(a)),  # Pandas Datetime Series
+        lambda a: pd.DatetimeIndex(a).values,  # Numpy datetime64 array
+    ]
+)
+def datetime_array(request, datetime_value):
+    return request.param([datetime_value] * 3)
+
+
+# Encoding tests
+def test_graph_object_input(engine):
+    scatter = go.Scatter(x=[1, 2, 3], y=np.array([4, 5, 6]))
+    result = pio.to_plotly_json(scatter, engine=engine)
+    assert result == """{"type":"scatter","x":[1,2,3],"y":[4,5,6]}"""
+
+
+def test_numeric_numpy_encoding(numeric_numpy_array, engine, pretty):
+    value = build_test_dict(numeric_numpy_array)
+    result = pio.to_plotly_json(value, engine=engine, pretty=pretty)
+
+    array_str = to_json_test(numeric_numpy_array.tolist())
+    expected = build_test_dict_string(array_str, pretty=pretty)
+    assert result == expected
+
+
+def test_object_numpy_encoding(object_numpy_array, engine, pretty):
+    value = build_test_dict(object_numpy_array)
+    result = pio.to_plotly_json(value, engine=engine, pretty=pretty)
+
+    array_str = to_json_test(object_numpy_array.tolist())
+    expected = build_test_dict_string(array_str)
+    assert result == expected
+
+
+def test_datetime(datetime_value, engine, pretty):
+    if engine == "legacy":
+        pytest.skip("legacy encoder doesn't strip timezone from scalar datetimes")
+
+    value = build_test_dict(datetime_value)
+    result = pio.to_plotly_json(value, engine=engine, pretty=pretty)
+    expected = build_test_dict_string('"{}"'.format(isoformat_test(datetime_value)))
+    assert result == expected
+
+
+def test_datetime_arrays(datetime_array, engine, pretty):
+    value = build_test_dict(datetime_array)
+    result = pio.to_plotly_json(value, engine=engine)
+
+    if isinstance(datetime_array, pd.Series):
+        dt_values = [d.isoformat() for d in datetime_array.dt.to_pydatetime().tolist()]
+    elif isinstance(datetime_array, pd.DatetimeIndex):
+        dt_values = [d.isoformat() for d in datetime_array.to_pydatetime().tolist()]
+    else:  # numpy datetime64 array
+        dt_values = datetime_array.tolist()
+
+    array_str = to_json_test(dt_values)
+    expected = build_test_dict_string(array_str)
+    assert result == expected

From 101ba8512a9c7577a2bec19cb38813255ba7e3d2 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 11:22:23 -0500
Subject: [PATCH 09/43] add testing of from_plotly_json

---
 packages/python/plotly/plotly/io/_json.py     |  8 +++----
 .../tests/test_io/test_to_from_plotly_json.py | 24 +++++++++++++++++--
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index d809e702cc9..21604698432 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -270,8 +270,8 @@ def from_plotly_json(value, engine=None):
 
     Parameters
     ----------
-    value: str
-        A JSON string
+    value: str or bytes
+        A JSON string or bytes object
 
     engine: str (default None)
         The JSON decoding engine to use. One of:
@@ -330,8 +330,8 @@ def from_json(value, output_type="Figure", skip_invalid=False, engine=None):
 
     Parameters
     ----------
-    value: str
-        String containing the JSON representation of a figure
+    value: str or bytes
+        String or bytes object containing the JSON representation of a figure
 
     output_type: type or str (default 'Figure')
         The output figure type or type name.
diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
index 2cc571488fe..21df0384196 100644
--- a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
+++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
@@ -5,6 +5,7 @@
 import pytest
 import json
 import datetime
+import sys
 from pytz import timezone
 
 eastern = timezone("US/Eastern")
@@ -46,6 +47,19 @@ def build_test_dict_string(value_string, pretty=False):
         return """{"a":%s,"b":[3,%s],"c":{"Z":%s}}""" % tuple([value_string] * 3)
 
 
+def check_roundtrip(value, engine, pretty):
+    encoded = pio.to_plotly_json(value, engine=engine, pretty=pretty)
+    decoded = pio.from_plotly_json(encoded, engine=engine)
+    reencoded = pio.to_plotly_json(decoded, engine=engine, pretty=pretty)
+    assert encoded == reencoded
+
+    # Check from_plotly_json with bytes on Python 3
+    if sys.version_info.major == 3:
+        encoded_bytes = encoded.encode("utf8")
+        decoded_from_bytes = pio.from_plotly_json(encoded_bytes, engine=engine)
+        assert decoded == decoded_from_bytes
+
+
 # Fixtures
 @pytest.fixture(scope="module", params=["json", "orjson", "legacy", "auto"])
 def engine(request):
@@ -99,10 +113,12 @@ def datetime_array(request, datetime_value):
 
 
 # Encoding tests
-def test_graph_object_input(engine):
+def test_graph_object_input(engine, pretty):
     scatter = go.Scatter(x=[1, 2, 3], y=np.array([4, 5, 6]))
     result = pio.to_plotly_json(scatter, engine=engine)
-    assert result == """{"type":"scatter","x":[1,2,3],"y":[4,5,6]}"""
+    expected = """{"type":"scatter","x":[1,2,3],"y":[4,5,6]}"""
+    assert result == expected
+    check_roundtrip(result, engine=engine, pretty=pretty)
 
 
 def test_numeric_numpy_encoding(numeric_numpy_array, engine, pretty):
@@ -112,6 +128,7 @@ def test_numeric_numpy_encoding(numeric_numpy_array, engine, pretty):
     array_str = to_json_test(numeric_numpy_array.tolist())
     expected = build_test_dict_string(array_str, pretty=pretty)
     assert result == expected
+    check_roundtrip(result, engine=engine, pretty=pretty)
 
 
 def test_object_numpy_encoding(object_numpy_array, engine, pretty):
@@ -121,6 +138,7 @@ def test_object_numpy_encoding(object_numpy_array, engine, pretty):
     array_str = to_json_test(object_numpy_array.tolist())
     expected = build_test_dict_string(array_str)
     assert result == expected
+    check_roundtrip(result, engine=engine, pretty=pretty)
 
 
 def test_datetime(datetime_value, engine, pretty):
@@ -131,6 +149,7 @@ def test_datetime(datetime_value, engine, pretty):
     result = pio.to_plotly_json(value, engine=engine, pretty=pretty)
     expected = build_test_dict_string('"{}"'.format(isoformat_test(datetime_value)))
     assert result == expected
+    check_roundtrip(result, engine=engine, pretty=pretty)
 
 
 def test_datetime_arrays(datetime_array, engine, pretty):
@@ -147,3 +166,4 @@ def test_datetime_arrays(datetime_array, engine, pretty):
     array_str = to_json_test(dt_values)
     expected = build_test_dict_string(array_str)
     assert result == expected
+    check_roundtrip(result, engine=engine, pretty=pretty)

From 67d3670e8392415d8a6ef5200e08bdef9b95ec9f Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 11:27:41 -0500
Subject: [PATCH 10/43] Better error message when orjson not installed and
 orjson engine requested

---
 packages/python/plotly/plotly/io/_json.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 21604698432..52c0363e99c 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -32,12 +32,16 @@ def default_engine(self, val):
             )
 
         if val == "orjson":
-            orjson = get_module("orjson")
-            if orjson is None:
-                raise ValueError("The orjson engine requires the orjson package")
+            self.validate_orjson()
 
         self._default_engine = val
 
+    @classmethod
+    def validate_orjson(cls):
+        orjson = get_module("orjson")
+        if orjson is None:
+            raise ValueError("The orjson engine requires the orjson package")
+
 
 config = JsonConfig()
 
@@ -106,8 +110,6 @@ def to_plotly_json(plotly_object, pretty=False, engine=None):
         "image": get_module("PIL.Image", should_load=False),
     }
 
-    orjson = get_module("orjson", should_load=True)
-
     # Dump to a JSON string and return
     # --------------------------------
     if engine in ("json", "legacy"):
@@ -148,6 +150,7 @@ def to_plotly_json(plotly_object, pretty=False, engine=None):
 
             return json.dumps(plotly_object, cls=PlotlyJSONEncoder, **opts)
     elif engine == "orjson":
+        JsonConfig.validate_orjson()
         opts = orjson.OPT_SORT_KEYS | orjson.OPT_SERIALIZE_NUMPY
 
         if pretty:
@@ -287,6 +290,8 @@ def from_plotly_json(value, engine=None):
     -------
     dict
     """
+    orjson = get_module("orjson", should_load=True)
+
     # Validate value
     # --------------
     if not isinstance(value, (string_types, bytes)):
@@ -298,8 +303,6 @@ def from_plotly_json(value, engine=None):
             )
         )
 
-    orjson = get_module("orjson", should_load=True)
-
     # Determine json engine
     if engine is None:
         engine = config.default_engine
@@ -313,6 +316,7 @@ def from_plotly_json(value, engine=None):
         raise ValueError("Invalid json engine: %s" % engine)
 
     if engine == "orjson":
+        JsonConfig.validate_orjson()
         # orjson handles bytes input natively
         value_dict = orjson.loads(value)
     else:

From 02c00daf2865a8c27d38a9680739ba150f5d2ade Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 11:28:41 -0500
Subject: [PATCH 11/43] Add orjson as optional testing dependency

---
 packages/python/plotly/tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/python/plotly/tox.ini b/packages/python/plotly/tox.ini
index 72169192a08..4e050e0a441 100644
--- a/packages/python/plotly/tox.ini
+++ b/packages/python/plotly/tox.ini
@@ -75,6 +75,7 @@ deps=
     optional: matplotlib==2.2.3
     optional: scikit-image==0.14.4
     optional: kaleido
+    optional: orjson==3.4.6
 
 ; CORE ENVIRONMENTS
 [testenv:py27-core]

From 99ea6a16e68cb7be7bbf871309f89cba608623b5 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 11:45:40 -0500
Subject: [PATCH 12/43] Replace Python 3.5 CI tests with 3.8

---
 .circleci/config.yml           | 60 +++++++++++++++++-----------------
 packages/python/plotly/tox.ini | 26 +++++++--------
 2 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index fae56c4ceae..e02d94f9c47 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -31,11 +31,11 @@ jobs:
           command: "cd packages/python/plotly; tox -e py27-core"
           no_output_timeout: 20m
 
-  python-3.5-core:
+  python-3.6-core:
     docker:
-      - image: circleci/python:3.5-stretch-node-browsers
+      - image: circleci/python:3.6-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_35: python3.5
+          PLOTLY_TOX_PYTHON_36: python3.6
 
     steps:
       - checkout
@@ -44,14 +44,14 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py35-core"
+          command: "cd packages/python/plotly; tox -e py36-core"
           no_output_timeout: 20m
 
-  python-3.6-core:
+  python-3.7-core:
     docker:
-      - image: circleci/python:3.6-stretch-node-browsers
+      - image: circleci/python:3.7-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_36: python3.6
+          PLOTLY_TOX_PYTHON_37: python3.7
 
     steps:
       - checkout
@@ -60,14 +60,14 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py36-core"
+          command: "cd packages/python/plotly; tox -e py37-core"
           no_output_timeout: 20m
 
-  python-3.7-core:
+  python-3.8-core:
     docker:
-      - image: circleci/python:3.7-stretch-node-browsers
+      - image: circleci/python:3.8-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_37: python3.7
+          PLOTLY_TOX_PYTHON_38: python3.8
 
     steps:
       - checkout
@@ -76,7 +76,7 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py37-core"
+          command: "cd packages/python/plotly; tox -e py38-core"
           no_output_timeout: 20m
 
   python-3.7-percy:
@@ -128,11 +128,11 @@ jobs:
           command: "cd packages/python/plotly; tox -e py27-optional"
           no_output_timeout: 20m
 
-  python-3.5-optional:
+  python-3.6-optional:
     docker:
-      - image: circleci/python:3.5-stretch-node-browsers
+      - image: circleci/python:3.6-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_35: python3.5
+          PLOTLY_TOX_PYTHON_36: python3.6
 
     steps:
       - checkout
@@ -141,14 +141,14 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py35-optional"
+          command: "cd packages/python/plotly; tox -e py36-optional"
           no_output_timeout: 20m
 
-  python-3.6-optional:
+  python-3.7-optional:
     docker:
-      - image: circleci/python:3.6-stretch-node-browsers
+      - image: circleci/python:3.7-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_36: python3.6
+          PLOTLY_TOX_PYTHON_37: python3.7
 
     steps:
       - checkout
@@ -157,14 +157,14 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py36-optional"
+          command: "cd packages/python/plotly; tox -e py37-optional"
           no_output_timeout: 20m
 
-  python-3.7-optional:
+  python-3.8-optional:
     docker:
-      - image: circleci/python:3.7-stretch-node-browsers
+      - image: circleci/python:3.8-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_37: python3.7
+          PLOTLY_TOX_PYTHON_38: python3.8
 
     steps:
       - checkout
@@ -173,7 +173,7 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py37-optional"
+          command: "cd packages/python/plotly; tox -e py38-optional"
           no_output_timeout: 20m
 
   # Plot.ly
@@ -255,23 +255,23 @@ jobs:
       - store_artifacts:
           path: plotly/tests/test_orca/images/linux/failed
 
-  python-3-5-orca:
+  python-3-6-orca:
     docker:
       - image: circleci/node:10.9-stretch-browsers
         environment:
-          PYTHON_VERSION: 3.5
+          PYTHON_VERSION: 3.6
 
     steps:
       - checkout
       - restore_cache:
           keys:
-            - conda-35-v1-{{ checksum ".circleci/create_conda_optional_env.sh" }}
+            - conda-36-v1-{{ checksum ".circleci/create_conda_optional_env.sh" }}
       - run:
           name: Create conda environment
           command: .circleci/create_conda_optional_env.sh
 
       - save_cache:
-          key: conda-35-v1-{{ checksum ".circleci/create_conda_optional_env.sh" }}
+          key: conda-36-v1-{{ checksum ".circleci/create_conda_optional_env.sh" }}
           paths:
             - /home/circleci/miniconda/
       - run:
@@ -518,14 +518,14 @@ workflows:
   build:
     jobs:
       - python-2.7-core
-      - python-3.5-core
       - python-3.6-core
       - python-3.7-core
+      - python-3.8-core
       - python-3.7-percy
       - python-2.7-optional
-      - python-3.5-optional
       - python-3.6-optional
       - python-3.7-optional
+      - python-3.8-optional
       - python-3.7-plot_ly
       - python-2-7-orca
       - python-3-7-orca
diff --git a/packages/python/plotly/tox.ini b/packages/python/plotly/tox.ini
index 4e050e0a441..ecca2b44dff 100644
--- a/packages/python/plotly/tox.ini
+++ b/packages/python/plotly/tox.ini
@@ -36,7 +36,7 @@
 [tox]
 ; The py{A,B,C}-{X,Y} generates a matrix of envs:
 ;   pyA-X,pyA-Y,pyB-X,pyB-Y,pyC-X,pyC-Y
-envlist = py{27,34,35,36,37}-{core,optional},py{27,34,37}
+envlist = py{27,36,37,38}-{core,optional},py{27,37}
 
 ; Note that envs can be targeted by deps using the <target>: dep syntax.
 ; Only one dep is allowed per line as of the time of writing. The <target>
@@ -84,12 +84,6 @@ commands=
     python --version
     pytest {posargs} plotly/tests/test_core
 
-[testenv:py35-core]
-basepython={env:PLOTLY_TOX_PYTHON_35:}
-commands=
-    python --version
-    pytest {posargs} plotly/tests/test_core
-
 [testenv:py36-core]
 basepython={env:PLOTLY_TOX_PYTHON_36:}
 commands=
@@ -104,6 +98,12 @@ commands=
     pytest {posargs} -x test_init/test_dependencies_not_imported.py
     pytest {posargs} -x test_init/test_lazy_imports.py
 
+[testenv:py38-core]
+basepython={env:PLOTLY_TOX_PYTHON_38:}
+commands=
+    python --version
+    pytest {posargs} plotly/tests/test_core
+
 ; OPTIONAL ENVIRONMENTS
 ;[testenv:py27-optional]
 ;basepython={env:PLOTLY_TOX_PYTHON_27:}
@@ -124,8 +124,8 @@ commands=
     pytest _plotly_utils/tests/
     pytest plotly/tests/test_io
 
-[testenv:py35-optional]
-basepython={env:PLOTLY_TOX_PYTHON_35:}
+[testenv:py36-optional]
+basepython={env:PLOTLY_TOX_PYTHON_36:}
 commands=
     python --version
     pytest {posargs} plotly/tests/test_core
@@ -133,8 +133,8 @@ commands=
     pytest _plotly_utils/tests/
     pytest plotly/tests/test_io
 
-[testenv:py36-optional]
-basepython={env:PLOTLY_TOX_PYTHON_36:}
+[testenv:py37-optional]
+basepython={env:PLOTLY_TOX_PYTHON_37:}
 commands=
     python --version
     pytest {posargs} plotly/tests/test_core
@@ -142,8 +142,8 @@ commands=
     pytest _plotly_utils/tests/
     pytest plotly/tests/test_io
 
-[testenv:py37-optional]
-basepython={env:PLOTLY_TOX_PYTHON_37:}
+[testenv:py38-optional]
+basepython={env:PLOTLY_TOX_PYTHON_38:}
 commands=
     python --version
     pytest {posargs} plotly/tests/test_core

From d44ec2645d1921e4dcd924e26db16e38cc2bdf6a Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 11:49:04 -0500
Subject: [PATCH 13/43] Try only install orjson with Python 3.6+

---
 packages/python/plotly/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/python/plotly/tox.ini b/packages/python/plotly/tox.ini
index ecca2b44dff..b1bf76bd384 100644
--- a/packages/python/plotly/tox.ini
+++ b/packages/python/plotly/tox.ini
@@ -75,7 +75,7 @@ deps=
     optional: matplotlib==2.2.3
     optional: scikit-image==0.14.4
     optional: kaleido
-    optional: orjson==3.4.6
+    optional: orjson==3.4.6;python_version<"3.5"
 
 ; CORE ENVIRONMENTS
 [testenv:py27-core]

From b7d842270b7351a8030f05bab8d7672182666f78 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 11:50:30 -0500
Subject: [PATCH 14/43] Don't test orjson engine when orjson not installed

---
 .../plotly/tests/test_io/test_to_from_plotly_json.py  | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
index 21df0384196..436dc924d22 100644
--- a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
+++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
@@ -1,12 +1,15 @@
+import pytest
 import plotly.io.json as pio
 import plotly.graph_objects as go
 import numpy as np
 import pandas as pd
-import pytest
 import json
 import datetime
 import sys
 from pytz import timezone
+from _plotly_utils.optional_imports import get_module
+
+orjson = get_module("orjson")
 
 eastern = timezone("US/Eastern")
 
@@ -61,6 +64,12 @@ def check_roundtrip(value, engine, pretty):
 
 
 # Fixtures
+if orjson is not None:
+    engines = ["json", "orjson", "legacy", "auto"]
+else:
+    engines = ["json", "legacy", "auto"]
+
+
 @pytest.fixture(scope="module", params=["json", "orjson", "legacy", "auto"])
 def engine(request):
     return request.param

From ddcd6f5c9f8fa43b53c3304e6b49a3a467191d02 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 11:53:17 -0500
Subject: [PATCH 15/43] Try new 3.8.7 docker image since prior guess doesn't
 exist

---
 .circleci/config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e02d94f9c47..9de385ee11f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -65,7 +65,7 @@ jobs:
 
   python-3.8-core:
     docker:
-      - image: circleci/python:3.8-stretch-node-browsers
+      - image: circleci/python:3.8.7
         environment:
           PLOTLY_TOX_PYTHON_38: python3.8
 
@@ -162,7 +162,7 @@ jobs:
 
   python-3.8-optional:
     docker:
-      - image: circleci/python:3.8-stretch-node-browsers
+      - image: circleci/python:3.8.7
         environment:
           PLOTLY_TOX_PYTHON_38: python3.8
 

From 33359f3bdd68de8ea7d2bf689f233ae3670d9402 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 11:55:19 -0500
Subject: [PATCH 16/43] greater than!

---
 packages/python/plotly/tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/python/plotly/tox.ini b/packages/python/plotly/tox.ini
index b1bf76bd384..72db48519f5 100644
--- a/packages/python/plotly/tox.ini
+++ b/packages/python/plotly/tox.ini
@@ -75,7 +75,7 @@ deps=
     optional: matplotlib==2.2.3
     optional: scikit-image==0.14.4
     optional: kaleido
-    optional: orjson==3.4.6;python_version<"3.5"
+    optional: orjson==3.4.6;python_version>"3.5"
 
 ; CORE ENVIRONMENTS
 [testenv:py27-core]

From c7c18197f2f38b091c78e6481808ac69fc28a899 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 11:59:17 -0500
Subject: [PATCH 17/43] Bump scikit image version for Python 3.8 compatibility

---
 packages/python/plotly/tox.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/python/plotly/tox.ini b/packages/python/plotly/tox.ini
index 72db48519f5..d97afcfb936 100644
--- a/packages/python/plotly/tox.ini
+++ b/packages/python/plotly/tox.ini
@@ -73,7 +73,8 @@ deps=
     optional: geopandas==0.3.0
     optional: pyshp==1.2.10
     optional: matplotlib==2.2.3
-    optional: scikit-image==0.14.4
+    optional: scikit-image==0.14.4;python_version<"3.0"
+    optional: scikit-image==0.18.1;python_version>"3.5"
     optional: kaleido
     optional: orjson==3.4.6;python_version>"3.5"
 

From a8d52abb3fccfdfadd34a11538550d53bcebbaa2 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 12:04:05 -0500
Subject: [PATCH 18/43] Try to help Python 2 from getting confused about which
 json module to import

---
 packages/python/plotly/plotly/io/_html.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_html.py b/packages/python/plotly/plotly/io/_html.py
index cb90de32f98..2d8511ab1c0 100644
--- a/packages/python/plotly/plotly/io/_html.py
+++ b/packages/python/plotly/plotly/io/_html.py
@@ -1,14 +1,16 @@
 import uuid
-import json
 import os
 import webbrowser
 
 import six
 
+from _plotly_utils.optional_imports import get_module
 from plotly.io._utils import validate_coerce_fig_to_dict
 from plotly.offline.offline import _get_jconfig, get_plotlyjs
 from plotly import utils
 
+_json = get_module("json")
+
 
 # Build script to set global PlotlyConfig object. This must execute before
 # plotly.js is loaded.
@@ -134,15 +136,15 @@ def to_html(
     plotdivid = str(uuid.uuid4())
 
     # ## Serialize figure ##
-    jdata = json.dumps(
+    jdata = _json.dumps(
         fig_dict.get("data", []), cls=utils.PlotlyJSONEncoder, sort_keys=True
     )
-    jlayout = json.dumps(
+    jlayout = _json.dumps(
         fig_dict.get("layout", {}), cls=utils.PlotlyJSONEncoder, sort_keys=True
     )
 
     if fig_dict.get("frames", None):
-        jframes = json.dumps(fig_dict.get("frames", []), cls=utils.PlotlyJSONEncoder)
+        jframes = _json.dumps(fig_dict.get("frames", []), cls=utils.PlotlyJSONEncoder)
     else:
         jframes = None
 
@@ -218,7 +220,7 @@ def to_html(
 
         if auto_play:
             if animation_opts:
-                animation_opts_arg = ", " + json.dumps(animation_opts)
+                animation_opts_arg = ", " + _json.dumps(animation_opts)
             else:
                 animation_opts_arg = ""
             then_animate = """.then(function(){{
@@ -228,7 +230,7 @@ def to_html(
             )
 
     # Serialize config dict to JSON
-    jconfig = json.dumps(config)
+    jconfig = _json.dumps(config)
 
     script = """\
                 if (document.getElementById("{id}")) {{\

From 619838f7d9f13d358b903200cf6777945d8def2b Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 12:06:30 -0500
Subject: [PATCH 19/43] Update pandas for Python 3

---
 packages/python/plotly/tox.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/python/plotly/tox.ini b/packages/python/plotly/tox.ini
index d97afcfb936..4fecfc7dfe6 100644
--- a/packages/python/plotly/tox.ini
+++ b/packages/python/plotly/tox.ini
@@ -57,7 +57,8 @@ deps=
     pytz==2016.10
     retrying==1.3.3
     pytest==3.5.1
-    pandas==0.24.2
+    pandas==0.24.2;python_version<"3.0"
+    pandas==1.2.0;python_version>"3.5"
     xarray==0.10.9
     statsmodels==0.10.2
     pillow==5.2.0

From 7c7a272fd4c94c42f3db79706f9a301fe1e2047c Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 12:09:22 -0500
Subject: [PATCH 20/43] Revert 3.8 CI updates. Too much for this PR

---
 .circleci/config.yml           | 60 +++++++++++++++++-----------------
 packages/python/plotly/tox.ini | 32 +++++++++---------
 2 files changed, 45 insertions(+), 47 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9de385ee11f..fae56c4ceae 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -31,11 +31,11 @@ jobs:
           command: "cd packages/python/plotly; tox -e py27-core"
           no_output_timeout: 20m
 
-  python-3.6-core:
+  python-3.5-core:
     docker:
-      - image: circleci/python:3.6-stretch-node-browsers
+      - image: circleci/python:3.5-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_36: python3.6
+          PLOTLY_TOX_PYTHON_35: python3.5
 
     steps:
       - checkout
@@ -44,14 +44,14 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py36-core"
+          command: "cd packages/python/plotly; tox -e py35-core"
           no_output_timeout: 20m
 
-  python-3.7-core:
+  python-3.6-core:
     docker:
-      - image: circleci/python:3.7-stretch-node-browsers
+      - image: circleci/python:3.6-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_37: python3.7
+          PLOTLY_TOX_PYTHON_36: python3.6
 
     steps:
       - checkout
@@ -60,14 +60,14 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py37-core"
+          command: "cd packages/python/plotly; tox -e py36-core"
           no_output_timeout: 20m
 
-  python-3.8-core:
+  python-3.7-core:
     docker:
-      - image: circleci/python:3.8.7
+      - image: circleci/python:3.7-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_38: python3.8
+          PLOTLY_TOX_PYTHON_37: python3.7
 
     steps:
       - checkout
@@ -76,7 +76,7 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py38-core"
+          command: "cd packages/python/plotly; tox -e py37-core"
           no_output_timeout: 20m
 
   python-3.7-percy:
@@ -128,11 +128,11 @@ jobs:
           command: "cd packages/python/plotly; tox -e py27-optional"
           no_output_timeout: 20m
 
-  python-3.6-optional:
+  python-3.5-optional:
     docker:
-      - image: circleci/python:3.6-stretch-node-browsers
+      - image: circleci/python:3.5-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_36: python3.6
+          PLOTLY_TOX_PYTHON_35: python3.5
 
     steps:
       - checkout
@@ -141,14 +141,14 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py36-optional"
+          command: "cd packages/python/plotly; tox -e py35-optional"
           no_output_timeout: 20m
 
-  python-3.7-optional:
+  python-3.6-optional:
     docker:
-      - image: circleci/python:3.7-stretch-node-browsers
+      - image: circleci/python:3.6-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_37: python3.7
+          PLOTLY_TOX_PYTHON_36: python3.6
 
     steps:
       - checkout
@@ -157,14 +157,14 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py37-optional"
+          command: "cd packages/python/plotly; tox -e py36-optional"
           no_output_timeout: 20m
 
-  python-3.8-optional:
+  python-3.7-optional:
     docker:
-      - image: circleci/python:3.8.7
+      - image: circleci/python:3.7-stretch-node-browsers
         environment:
-          PLOTLY_TOX_PYTHON_38: python3.8
+          PLOTLY_TOX_PYTHON_37: python3.7
 
     steps:
       - checkout
@@ -173,7 +173,7 @@ jobs:
           command: "sudo pip install tox"
       - run:
           name: Test with tox
-          command: "cd packages/python/plotly; tox -e py38-optional"
+          command: "cd packages/python/plotly; tox -e py37-optional"
           no_output_timeout: 20m
 
   # Plot.ly
@@ -255,23 +255,23 @@ jobs:
       - store_artifacts:
           path: plotly/tests/test_orca/images/linux/failed
 
-  python-3-6-orca:
+  python-3-5-orca:
     docker:
       - image: circleci/node:10.9-stretch-browsers
         environment:
-          PYTHON_VERSION: 3.6
+          PYTHON_VERSION: 3.5
 
     steps:
       - checkout
       - restore_cache:
           keys:
-            - conda-36-v1-{{ checksum ".circleci/create_conda_optional_env.sh" }}
+            - conda-35-v1-{{ checksum ".circleci/create_conda_optional_env.sh" }}
       - run:
           name: Create conda environment
           command: .circleci/create_conda_optional_env.sh
 
       - save_cache:
-          key: conda-36-v1-{{ checksum ".circleci/create_conda_optional_env.sh" }}
+          key: conda-35-v1-{{ checksum ".circleci/create_conda_optional_env.sh" }}
           paths:
             - /home/circleci/miniconda/
       - run:
@@ -518,14 +518,14 @@ workflows:
   build:
     jobs:
       - python-2.7-core
+      - python-3.5-core
       - python-3.6-core
       - python-3.7-core
-      - python-3.8-core
       - python-3.7-percy
       - python-2.7-optional
+      - python-3.5-optional
       - python-3.6-optional
       - python-3.7-optional
-      - python-3.8-optional
       - python-3.7-plot_ly
       - python-2-7-orca
       - python-3-7-orca
diff --git a/packages/python/plotly/tox.ini b/packages/python/plotly/tox.ini
index 4fecfc7dfe6..5b4a8345545 100644
--- a/packages/python/plotly/tox.ini
+++ b/packages/python/plotly/tox.ini
@@ -36,7 +36,7 @@
 [tox]
 ; The py{A,B,C}-{X,Y} generates a matrix of envs:
 ;   pyA-X,pyA-Y,pyB-X,pyB-Y,pyC-X,pyC-Y
-envlist = py{27,36,37,38}-{core,optional},py{27,37}
+envlist = py{27,34,35,36,37}-{core,optional},py{27,34,37}
 
 ; Note that envs can be targeted by deps using the <target>: dep syntax.
 ; Only one dep is allowed per line as of the time of writing. The <target>
@@ -57,8 +57,7 @@ deps=
     pytz==2016.10
     retrying==1.3.3
     pytest==3.5.1
-    pandas==0.24.2;python_version<"3.0"
-    pandas==1.2.0;python_version>"3.5"
+    pandas==0.24.2
     xarray==0.10.9
     statsmodels==0.10.2
     pillow==5.2.0
@@ -74,8 +73,7 @@ deps=
     optional: geopandas==0.3.0
     optional: pyshp==1.2.10
     optional: matplotlib==2.2.3
-    optional: scikit-image==0.14.4;python_version<"3.0"
-    optional: scikit-image==0.18.1;python_version>"3.5"
+    optional: scikit-image==0.14.4
     optional: kaleido
     optional: orjson==3.4.6;python_version>"3.5"
 
@@ -86,6 +84,12 @@ commands=
     python --version
     pytest {posargs} plotly/tests/test_core
 
+[testenv:py35-core]
+basepython={env:PLOTLY_TOX_PYTHON_35:}
+commands=
+    python --version
+    pytest {posargs} plotly/tests/test_core
+
 [testenv:py36-core]
 basepython={env:PLOTLY_TOX_PYTHON_36:}
 commands=
@@ -100,12 +104,6 @@ commands=
     pytest {posargs} -x test_init/test_dependencies_not_imported.py
     pytest {posargs} -x test_init/test_lazy_imports.py
 
-[testenv:py38-core]
-basepython={env:PLOTLY_TOX_PYTHON_38:}
-commands=
-    python --version
-    pytest {posargs} plotly/tests/test_core
-
 ; OPTIONAL ENVIRONMENTS
 ;[testenv:py27-optional]
 ;basepython={env:PLOTLY_TOX_PYTHON_27:}
@@ -126,8 +124,8 @@ commands=
     pytest _plotly_utils/tests/
     pytest plotly/tests/test_io
 
-[testenv:py36-optional]
-basepython={env:PLOTLY_TOX_PYTHON_36:}
+[testenv:py35-optional]
+basepython={env:PLOTLY_TOX_PYTHON_35:}
 commands=
     python --version
     pytest {posargs} plotly/tests/test_core
@@ -135,8 +133,8 @@ commands=
     pytest _plotly_utils/tests/
     pytest plotly/tests/test_io
 
-[testenv:py37-optional]
-basepython={env:PLOTLY_TOX_PYTHON_37:}
+[testenv:py36-optional]
+basepython={env:PLOTLY_TOX_PYTHON_36:}
 commands=
     python --version
     pytest {posargs} plotly/tests/test_core
@@ -144,8 +142,8 @@ commands=
     pytest _plotly_utils/tests/
     pytest plotly/tests/test_io
 
-[testenv:py38-optional]
-basepython={env:PLOTLY_TOX_PYTHON_38:}
+[testenv:py37-optional]
+basepython={env:PLOTLY_TOX_PYTHON_37:}
 commands=
     python --version
     pytest {posargs} plotly/tests/test_core

From 17087037a952040d23844444d58173d318c35e67 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 12:20:22 -0500
Subject: [PATCH 21/43] Doh

---
 .../plotly/plotly/tests/test_io/test_to_from_plotly_json.py     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
index 436dc924d22..023fa929072 100644
--- a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
+++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
@@ -70,7 +70,7 @@ def check_roundtrip(value, engine, pretty):
     engines = ["json", "legacy", "auto"]
 
 
-@pytest.fixture(scope="module", params=["json", "orjson", "legacy", "auto"])
+@pytest.fixture(scope="module", params=engines)
 def engine(request):
     return request.param
 

From 66cab10b87ff11d923c1b62559d091c7e10f128b Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 31 Dec 2020 15:35:03 -0500
Subject: [PATCH 22/43] Don't skip copying during serialization

---
 packages/python/plotly/plotly/io/_json.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 52c0363e99c..8a0fc1dc3f2 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -202,7 +202,7 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
     """
     # Validate figure
     # ---------------
-    fig_dict = validate_coerce_fig_to_dict(fig, validate, clone=False)
+    fig_dict = validate_coerce_fig_to_dict(fig, validate)
 
     # Remove trace uid
     # ----------------
@@ -455,9 +455,6 @@ def clean_to_json_compatible(obj, **kwargs):
 
     # Plotly
     try:
-        obj = obj.to_plotly_json(clone=False)
-    except (TypeError, NameError, ValueError):
-        # Try without clone for backward compatibility
         obj = obj.to_plotly_json()
     except AttributeError:
         pass

From 56a8945a1c3b57dfd286c784dab01471f7fc4e73 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 2 Jan 2021 09:02:38 -0500
Subject: [PATCH 23/43] Rename new JSON functions:     -
 pio.json.to_plotly_json -> pio.json.to_json_plotly     -
 pio.json.from_plotly_json -> pio.json.from_json_plotly

---
 packages/python/plotly/plotly/io/_json.py      | 16 ++++++++++------
 packages/python/plotly/plotly/io/json.py       |  4 ++--
 .../tests/test_io/test_to_from_plotly_json.py  | 18 +++++++++---------
 3 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 8a0fc1dc3f2..45e01faf3c9 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -58,7 +58,7 @@ def coerce_to_strict(const):
         return const
 
 
-def to_plotly_json(plotly_object, pretty=False, engine=None):
+def to_json_plotly(plotly_object, pretty=False, engine=None):
     """
     Convert a plotly/Dash object to a JSON string representation
 
@@ -198,7 +198,7 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
 
     See Also
     --------
-    to_plotly_json : Convert an arbitrary plotly graph_object or Dash component to JSON
+    to_json_plotly : Convert an arbitrary plotly graph_object or Dash component to JSON
     """
     # Validate figure
     # ---------------
@@ -210,7 +210,7 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
         for trace in fig_dict.get("data", []):
             trace.pop("uid", None)
 
-    return to_plotly_json(fig_dict, pretty=pretty, engine=engine)
+    return to_json_plotly(fig_dict, pretty=pretty, engine=engine)
 
 
 def write_json(fig, file, validate=True, pretty=False, remove_uids=True, engine=None):
@@ -267,7 +267,7 @@ def write_json(fig, file, validate=True, pretty=False, remove_uids=True, engine=
         file.write(json_str)
 
 
-def from_plotly_json(value, engine=None):
+def from_json_plotly(value, engine=None):
     """
     Parse JSON string using the specified JSON engine
 
@@ -289,6 +289,10 @@ def from_plotly_json(value, engine=None):
     Returns
     -------
     dict
+
+    See Also
+    --------
+    from_json_plotly : Parse JSON with plotly conventions into a dict
     """
     orjson = get_module("orjson", should_load=True)
 
@@ -297,7 +301,7 @@ def from_plotly_json(value, engine=None):
     if not isinstance(value, (string_types, bytes)):
         raise ValueError(
             """
-from_plotly_json requires a string or bytes argument but received value of type {typ}
+from_json_plotly requires a string or bytes argument but received value of type {typ}
     Received value: {value}""".format(
                 typ=type(value), value=value
             )
@@ -368,7 +372,7 @@ def from_json(value, output_type="Figure", skip_invalid=False, engine=None):
 
     # Decode JSON
     # -----------
-    fig_dict = from_plotly_json(value, engine=engine)
+    fig_dict = from_json_plotly(value, engine=engine)
 
     # Validate coerce output type
     # ---------------------------
diff --git a/packages/python/plotly/plotly/io/json.py b/packages/python/plotly/plotly/io/json.py
index 8f895dc81a7..ff83e960760 100644
--- a/packages/python/plotly/plotly/io/json.py
+++ b/packages/python/plotly/plotly/io/json.py
@@ -4,6 +4,6 @@
     from_json,
     read_json,
     config,
-    to_plotly_json,
-    from_plotly_json,
+    to_json_plotly,
+    from_json_plotly,
 )
diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
index 023fa929072..ac638ccb632 100644
--- a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
+++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
@@ -51,15 +51,15 @@ def build_test_dict_string(value_string, pretty=False):
 
 
 def check_roundtrip(value, engine, pretty):
-    encoded = pio.to_plotly_json(value, engine=engine, pretty=pretty)
-    decoded = pio.from_plotly_json(encoded, engine=engine)
-    reencoded = pio.to_plotly_json(decoded, engine=engine, pretty=pretty)
+    encoded = pio.to_json_plotly(value, engine=engine, pretty=pretty)
+    decoded = pio.from_json_plotly(encoded, engine=engine)
+    reencoded = pio.to_json_plotly(decoded, engine=engine, pretty=pretty)
     assert encoded == reencoded
 
     # Check from_plotly_json with bytes on Python 3
     if sys.version_info.major == 3:
         encoded_bytes = encoded.encode("utf8")
-        decoded_from_bytes = pio.from_plotly_json(encoded_bytes, engine=engine)
+        decoded_from_bytes = pio.from_json_plotly(encoded_bytes, engine=engine)
         assert decoded == decoded_from_bytes
 
 
@@ -124,7 +124,7 @@ def datetime_array(request, datetime_value):
 # Encoding tests
 def test_graph_object_input(engine, pretty):
     scatter = go.Scatter(x=[1, 2, 3], y=np.array([4, 5, 6]))
-    result = pio.to_plotly_json(scatter, engine=engine)
+    result = pio.to_json_plotly(scatter, engine=engine)
     expected = """{"type":"scatter","x":[1,2,3],"y":[4,5,6]}"""
     assert result == expected
     check_roundtrip(result, engine=engine, pretty=pretty)
@@ -132,7 +132,7 @@ def test_graph_object_input(engine, pretty):
 
 def test_numeric_numpy_encoding(numeric_numpy_array, engine, pretty):
     value = build_test_dict(numeric_numpy_array)
-    result = pio.to_plotly_json(value, engine=engine, pretty=pretty)
+    result = pio.to_json_plotly(value, engine=engine, pretty=pretty)
 
     array_str = to_json_test(numeric_numpy_array.tolist())
     expected = build_test_dict_string(array_str, pretty=pretty)
@@ -142,7 +142,7 @@ def test_numeric_numpy_encoding(numeric_numpy_array, engine, pretty):
 
 def test_object_numpy_encoding(object_numpy_array, engine, pretty):
     value = build_test_dict(object_numpy_array)
-    result = pio.to_plotly_json(value, engine=engine, pretty=pretty)
+    result = pio.to_json_plotly(value, engine=engine, pretty=pretty)
 
     array_str = to_json_test(object_numpy_array.tolist())
     expected = build_test_dict_string(array_str)
@@ -155,7 +155,7 @@ def test_datetime(datetime_value, engine, pretty):
         pytest.skip("legacy encoder doesn't strip timezone from scalar datetimes")
 
     value = build_test_dict(datetime_value)
-    result = pio.to_plotly_json(value, engine=engine, pretty=pretty)
+    result = pio.to_json_plotly(value, engine=engine, pretty=pretty)
     expected = build_test_dict_string('"{}"'.format(isoformat_test(datetime_value)))
     assert result == expected
     check_roundtrip(result, engine=engine, pretty=pretty)
@@ -163,7 +163,7 @@ def test_datetime(datetime_value, engine, pretty):
 
 def test_datetime_arrays(datetime_array, engine, pretty):
     value = build_test_dict(datetime_array)
-    result = pio.to_plotly_json(value, engine=engine)
+    result = pio.to_json_plotly(value, engine=engine)
 
     if isinstance(datetime_array, pd.Series):
         dt_values = [d.isoformat() for d in datetime_array.dt.to_pydatetime().tolist()]

From 0a51020d4f7f636194c88cdc716cb81b48c93f1b Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Sat, 2 Jan 2021 12:36:39 -0500
Subject: [PATCH 24/43] Ensure cleaned numpy arrays are contiguous

---
 packages/python/plotly/plotly/io/_json.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 45e01faf3c9..f38147b5435 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -479,7 +479,7 @@ def clean_to_json_compatible(obj, **kwargs):
             and isinstance(obj, np.ndarray)
             and obj.dtype.kind in ("b", "i", "u", "f")
         ):
-            return obj
+            return np.ascontiguousarray(obj)
 
     # pandas
     if pd is not None:
@@ -487,7 +487,7 @@ def clean_to_json_compatible(obj, **kwargs):
             return None
         elif isinstance(obj, (pd.Series, pd.DatetimeIndex)):
             if numpy_allowed and obj.dtype.kind in ("b", "i", "u", "f"):
-                return obj.values
+                return np.ascontiguousarray(obj.values)
             elif obj.dtype.kind == "M":
                 if isinstance(obj, pd.Series):
                     dt_values = obj.dt.to_pydatetime().tolist()

From 4e9d64ecdc4bf65aaaa84a58ee3dc91ff5e158a5 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 8 Jan 2021 12:00:16 -0500
Subject: [PATCH 25/43] Use to_json_plotly in html and orca logic

---
 packages/python/plotly/plotly/io/_html.py      | 11 ++++-------
 packages/python/plotly/plotly/io/_orca.py      |  3 ++-
 packages/python/plotly/templategen/__init__.py |  5 ++---
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_html.py b/packages/python/plotly/plotly/io/_html.py
index 2d8511ab1c0..6d281503c7d 100644
--- a/packages/python/plotly/plotly/io/_html.py
+++ b/packages/python/plotly/plotly/io/_html.py
@@ -128,6 +128,7 @@ def to_html(
     str
         Representation of figure as an HTML div string
     """
+    from plotly.io.json import to_json_plotly
 
     # ## Validate figure ##
     fig_dict = validate_coerce_fig_to_dict(fig, validate)
@@ -136,15 +137,11 @@ def to_html(
     plotdivid = str(uuid.uuid4())
 
     # ## Serialize figure ##
-    jdata = _json.dumps(
-        fig_dict.get("data", []), cls=utils.PlotlyJSONEncoder, sort_keys=True
-    )
-    jlayout = _json.dumps(
-        fig_dict.get("layout", {}), cls=utils.PlotlyJSONEncoder, sort_keys=True
-    )
+    jdata = to_json_plotly(fig_dict.get("data", []))
+    jlayout = to_json_plotly(fig_dict.get("layout", {}))
 
     if fig_dict.get("frames", None):
-        jframes = _json.dumps(fig_dict.get("frames", []), cls=utils.PlotlyJSONEncoder)
+        jframes = to_json_plotly(fig_dict.get("frames", []))
     else:
         jframes = None
 
diff --git a/packages/python/plotly/plotly/io/_orca.py b/packages/python/plotly/plotly/io/_orca.py
index 500674b6281..e77f9b6f6fc 100644
--- a/packages/python/plotly/plotly/io/_orca.py
+++ b/packages/python/plotly/plotly/io/_orca.py
@@ -1458,6 +1458,7 @@ def request_image_with_retrying(**kwargs):
     with retrying logic.
     """
     from requests import post
+    from plotly.io.json import to_json_plotly
 
     if config.server_url:
         server_url = config.server_url
@@ -1467,7 +1468,7 @@ def request_image_with_retrying(**kwargs):
         )
 
     request_params = {k: v for k, v, in kwargs.items() if v is not None}
-    json_str = json.dumps(request_params, cls=_plotly_utils.utils.PlotlyJSONEncoder)
+    json_str = to_json_plotly(request_params)
     response = post(server_url + "/", data=json_str)
 
     if response.status_code == 522:
diff --git a/packages/python/plotly/templategen/__init__.py b/packages/python/plotly/templategen/__init__.py
index 9dcc21a765b..bdcf8e10448 100644
--- a/packages/python/plotly/templategen/__init__.py
+++ b/packages/python/plotly/templategen/__init__.py
@@ -1,5 +1,4 @@
-from plotly.utils import PlotlyJSONEncoder
-import json
+from plotly.io.json import to_json_plotly
 import os
 from templategen.definitions import builders
 
@@ -20,4 +19,4 @@
             ),
             "w",
         ) as f:
-            plotly_schema = json.dump(template, f, cls=PlotlyJSONEncoder)
+            f.write(to_json_plotly(template))

From d4068dee52035fce702ee85036d516048d6ba90e Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 8 Jan 2021 12:26:39 -0500
Subject: [PATCH 26/43] Add orjson documentation dependency

---
 doc/requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/requirements.txt b/doc/requirements.txt
index 86e81a4467b..7e73edf8e8a 100644
--- a/doc/requirements.txt
+++ b/doc/requirements.txt
@@ -30,3 +30,4 @@ kaleido
 umap-learn
 pooch
 wget
+orjson

From 58b7192c8f0d01d1e910bec7c0563b2af8b61cc4 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 8 Jan 2021 13:08:16 -0500
Subject: [PATCH 27/43] Handle pandas Timestamp scalars in orjson engine Add
 test for Timestamp and lists of time values

---
 packages/python/plotly/plotly/io/_json.py     | 13 ++++++---
 .../tests/test_io/test_to_from_plotly_json.py | 27 ++++++++++++++++---
 2 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index f38147b5435..d642b50401b 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -490,9 +490,9 @@ def clean_to_json_compatible(obj, **kwargs):
                 return np.ascontiguousarray(obj.values)
             elif obj.dtype.kind == "M":
                 if isinstance(obj, pd.Series):
-                    dt_values = obj.dt.to_pydatetime().tolist()
+                    dt_values = obj.dt.tz_localize(None).dt.to_pydatetime().tolist()
                 else:  # DatetimeIndex
-                    dt_values = obj.to_pydatetime().tolist()
+                    dt_values = obj.tz_localize(None).to_pydatetime().tolist()
 
                 if not datetime_allowed:
                     # Note: We don't need to handle dropping timezones here because
@@ -514,13 +514,18 @@ def clean_to_json_compatible(obj, **kwargs):
         if np and isinstance(obj, np.datetime64):
             return str(obj)
     else:
+        res = None
         try:
             # Need to drop timezone for scalar datetimes. Don't need to convert
             # to string since engine can do that
-            return obj.replace(tzinfo=None)
-        except AttributeError:
+            res = obj.replace(tzinfo=None)
+            res = res.to_pydatetime()
+        except(TypeError, AttributeError):
             pass
 
+        if res is not None:
+            return res
+
     # Try .tolist() convertible, do not recurse inside
     try:
         return obj.tolist()
diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
index ac638ccb632..8a5f6315201 100644
--- a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
+++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
@@ -102,8 +102,10 @@ def object_numpy_array(request):
         datetime.datetime(2003, 7, 12, 8, 34, 22),
         datetime.datetime.now(),
         np.datetime64(datetime.datetime.utcnow()),
+        pd.Timestamp(datetime.datetime.now()),
         eastern.localize(datetime.datetime(2003, 7, 12, 8, 34, 22)),
         eastern.localize(datetime.datetime.now()),
+        pd.Timestamp(datetime.datetime.now(), tzinfo=eastern),
     ],
 )
 def datetime_value(request):
@@ -112,6 +114,7 @@ def datetime_value(request):
 
 @pytest.fixture(
     params=[
+        list,  # plain list of datetime values
         lambda a: pd.DatetimeIndex(a),  # Pandas DatetimeIndex
         lambda a: pd.Series(pd.DatetimeIndex(a)),  # Pandas Datetime Series
         lambda a: pd.DatetimeIndex(a).values,  # Numpy datetime64 array
@@ -162,13 +165,31 @@ def test_datetime(datetime_value, engine, pretty):
 
 
 def test_datetime_arrays(datetime_array, engine, pretty):
+    if engine == "legacy":
+        pytest.skip("legacy encoder doesn't strip timezone from datetimes arrays")
+
     value = build_test_dict(datetime_array)
     result = pio.to_json_plotly(value, engine=engine)
 
-    if isinstance(datetime_array, pd.Series):
-        dt_values = [d.isoformat() for d in datetime_array.dt.to_pydatetime().tolist()]
+    def to_str(v):
+        try:
+            v = v.replace(tzinfo=None)
+        except (TypeError, AttributeError):
+            pass
+
+        try:
+            v = v.isoformat(sep="T")
+        except (TypeError, AttributeError):
+            pass
+
+        return str(v)
+
+    if isinstance(datetime_array, list):
+        dt_values = [to_str(d) for d in datetime_array]
+    elif isinstance(datetime_array, pd.Series):
+        dt_values = [to_str(d) for d in datetime_array.dt.to_pydatetime().tolist()]
     elif isinstance(datetime_array, pd.DatetimeIndex):
-        dt_values = [d.isoformat() for d in datetime_array.to_pydatetime().tolist()]
+        dt_values = [to_str(d) for d in datetime_array.to_pydatetime().tolist()]
     else:  # numpy datetime64 array
         dt_values = datetime_array.tolist()
 

From 974fcba97b7038669848543e38b36efa39b7f292 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 8 Jan 2021 14:52:14 -0500
Subject: [PATCH 28/43] Rework date and string encoding, add and fix tests

---
 packages/python/plotly/plotly/io/_json.py     | 54 +++++++++----------
 .../tests/test_io/test_to_from_plotly_json.py | 18 ++++++-
 2 files changed, 44 insertions(+), 28 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index d642b50401b..39a674b700a 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -3,8 +3,7 @@
 from six import string_types
 import json
 import decimal
-import os
-
+import datetime
 
 from plotly.io._utils import validate_coerce_fig_to_dict, validate_coerce_output_type
 from _plotly_utils.optional_imports import get_module
@@ -474,12 +473,19 @@ def clean_to_json_compatible(obj, **kwargs):
     if np is not None:
         if obj is np.ma.core.masked:
             return float("nan")
-        elif (
-            numpy_allowed
-            and isinstance(obj, np.ndarray)
-            and obj.dtype.kind in ("b", "i", "u", "f")
-        ):
-            return np.ascontiguousarray(obj)
+        elif isinstance(obj, np.ndarray):
+            if numpy_allowed and obj.dtype.kind in ("b", "i", "u", "f"):
+                return np.ascontiguousarray(obj)
+            elif obj.dtype.kind == "M":
+                # datetime64 array
+                return np.datetime_as_string(obj).tolist()
+            elif obj.dtype.kind == "U":
+                return obj.tolist()
+            elif obj.dtype.kind == "O":
+                # Treat object array as a lists, continue processing
+                obj = obj.tolist()
+        elif isinstance(obj, np.datetime64):
+            return str(obj)
 
     # pandas
     if pd is not None:
@@ -496,35 +502,29 @@ def clean_to_json_compatible(obj, **kwargs):
 
                 if not datetime_allowed:
                     # Note: We don't need to handle dropping timezones here because
-                    # numpy's datetime64 doesn't support them and pandas's tolist()
-                    # doesn't preserve them.
+                    # numpy's datetime64 doesn't support them and pandas's tz_localize
+                    # above drops them.
                     for i in range(len(dt_values)):
                         dt_values[i] = dt_values[i].isoformat()
 
                 return dt_values
 
     # datetime and date
-    if not datetime_allowed:
-        try:
-            # Need to drop timezone for scalar datetimes
-            return obj.replace(tzinfo=None).isoformat()
-        except (TypeError, AttributeError):
-            pass
+    try:
+        # Need to drop timezone for scalar datetimes. Don't need to convert
+        # to string since engine can do that
+        obj = obj.replace(tzinfo=None)
+        obj = obj.to_pydatetime()
+    except(TypeError, AttributeError):
+        pass
 
-        if np and isinstance(obj, np.datetime64):
-            return str(obj)
-    else:
-        res = None
+    if not datetime_allowed:
         try:
-            # Need to drop timezone for scalar datetimes. Don't need to convert
-            # to string since engine can do that
-            res = obj.replace(tzinfo=None)
-            res = res.to_pydatetime()
+            return obj.isoformat()
         except(TypeError, AttributeError):
             pass
-
-        if res is not None:
-            return res
+    elif isinstance(obj, datetime.datetime):
+        return obj
 
     # Try .tolist() convertible, do not recurse inside
     try:
diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
index 8a5f6315201..a72e64f66e9 100644
--- a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
+++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
@@ -96,6 +96,11 @@ def object_numpy_array(request):
     return np.array(["a", 1, [2, 3]])
 
 
+@pytest.fixture(scope="module")
+def numpy_unicode_array(request):
+    return np.array(["A", "BB", "CCC"], dtype="U")
+
+
 @pytest.fixture(
     scope="module",
     params=[
@@ -118,6 +123,7 @@ def datetime_value(request):
         lambda a: pd.DatetimeIndex(a),  # Pandas DatetimeIndex
         lambda a: pd.Series(pd.DatetimeIndex(a)),  # Pandas Datetime Series
         lambda a: pd.DatetimeIndex(a).values,  # Numpy datetime64 array
+        lambda a: np.array(a, dtype="object"),  # Numpy object array of datetime
     ]
 )
 def datetime_array(request, datetime_value):
@@ -143,6 +149,16 @@ def test_numeric_numpy_encoding(numeric_numpy_array, engine, pretty):
     check_roundtrip(result, engine=engine, pretty=pretty)
 
 
+def test_numpy_unicode_encoding(numpy_unicode_array, engine, pretty):
+    value = build_test_dict(numpy_unicode_array)
+    result = pio.to_json_plotly(value, engine=engine, pretty=pretty)
+
+    array_str = to_json_test(numpy_unicode_array.tolist())
+    expected = build_test_dict_string(array_str)
+    assert result == expected
+    check_roundtrip(result, engine=engine, pretty=pretty)
+
+
 def test_object_numpy_encoding(object_numpy_array, engine, pretty):
     value = build_test_dict(object_numpy_array)
     result = pio.to_json_plotly(value, engine=engine, pretty=pretty)
@@ -191,7 +207,7 @@ def to_str(v):
     elif isinstance(datetime_array, pd.DatetimeIndex):
         dt_values = [to_str(d) for d in datetime_array.to_pydatetime().tolist()]
     else:  # numpy datetime64 array
-        dt_values = datetime_array.tolist()
+        dt_values = [to_str(d) for d in datetime_array]
 
     array_str = to_json_test(dt_values)
     expected = build_test_dict_string(array_str)

From a651a631d6bca33ef0becfc55e7d326841d7435b Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 8 Jan 2021 15:03:26 -0500
Subject: [PATCH 29/43] default JSON engine to "auto"

---
 packages/python/plotly/plotly/io/_json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 39a674b700a..4252aa40a91 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -16,7 +16,7 @@ class JsonConfig(object):
     _valid_engines = ("legacy", "json", "orjson", "auto")
 
     def __init__(self):
-        self._default_engine = "legacy"
+        self._default_engine = "auto"
 
     @property
     def default_engine(self):

From af1d88d48eeaa4757701a95bba1f8043fcfba773 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 8 Jan 2021 15:12:17 -0500
Subject: [PATCH 30/43] Fix expected JSON in html export (no spaces)

---
 .../plotly/tests/test_core/test_offline/test_offline.py     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/python/plotly/plotly/tests/test_core/test_offline/test_offline.py b/packages/python/plotly/plotly/tests/test_core/test_offline/test_offline.py
index 315a81b9417..88096e0a7e6 100644
--- a/packages/python/plotly/plotly/tests/test_core/test_offline/test_offline.py
+++ b/packages/python/plotly/plotly/tests/test_core/test_offline/test_offline.py
@@ -88,7 +88,7 @@ def _read_html(self, file_url):
             return f.read()
 
     def test_default_plot_generates_expected_html(self):
-        layout_json = _json.dumps(fig["layout"], cls=plotly.utils.PlotlyJSONEncoder)
+        layout_json = pio.json.to_json_plotly(fig["layout"])
 
         html = self._read_html(
             plotly.offline.plot(fig, auto_open=False, filename=html_filename)
@@ -98,8 +98,8 @@ def test_default_plot_generates_expected_html(self):
         # instead just make sure a few of the parts are in here?
         self.assertIn("Plotly.newPlot", html)  # plot command is in there
 
-        x_data = '"x": [1, 2, 3]'
-        y_data = '"y": [10, 20, 30]'
+        x_data = '"x":[1,2,3]'
+        y_data = '"y":[10,20,30]'
 
         self.assertTrue(x_data in html and y_data in html)  # data in there
         self.assertIn(layout_json, html)  # so is layout

From d51fd94fcf5908c614298a9fbd491158c24ecb0a Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 8 Jan 2021 15:16:56 -0500
Subject: [PATCH 31/43] blacken

---
 packages/python/plotly/plotly/io/_json.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 4252aa40a91..42e80e425fc 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -515,13 +515,13 @@ def clean_to_json_compatible(obj, **kwargs):
         # to string since engine can do that
         obj = obj.replace(tzinfo=None)
         obj = obj.to_pydatetime()
-    except(TypeError, AttributeError):
+    except (TypeError, AttributeError):
         pass
 
     if not datetime_allowed:
         try:
             return obj.isoformat()
-        except(TypeError, AttributeError):
+        except (TypeError, AttributeError):
             pass
     elif isinstance(obj, datetime.datetime):
         return obj

From 042c54c1b1c09ea6e17ec68788b20503546d52cc Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 8 Jan 2021 15:21:51 -0500
Subject: [PATCH 32/43] Fix expected JSON in matplotlylib test

---
 .../tests/test_optional/test_offline/test_offline.py     | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/packages/python/plotly/plotly/tests/test_optional/test_offline/test_offline.py b/packages/python/plotly/plotly/tests/test_optional/test_offline/test_offline.py
index d36934d1eae..3503feb6fc9 100644
--- a/packages/python/plotly/plotly/tests/test_optional/test_offline/test_offline.py
+++ b/packages/python/plotly/plotly/tests/test_optional/test_offline/test_offline.py
@@ -10,6 +10,7 @@
 import pytest
 
 import plotly
+import plotly.io as pio
 from plotly import optional_imports
 
 matplotlylib = optional_imports.get_module("plotly.matplotlylib")
@@ -76,12 +77,8 @@ def test_default_mpl_plot_generates_expected_html(self):
             data = figure["data"]
 
             layout = figure["layout"]
-            data_json = _json.dumps(
-                data, cls=plotly.utils.PlotlyJSONEncoder, sort_keys=True
-            )
-            layout_json = _json.dumps(
-                layout, cls=plotly.utils.PlotlyJSONEncoder, sort_keys=True
-            )
+            data_json = pio.json.to_json_plotly(data)
+            layout_json = pio.json.to_json_plotly(layout)
             html = self._read_html(plotly.offline.plot_mpl(fig))
 
             # blank out uid before comparisons

From ddc1b8fd238cd6ada576ba1bb32508caeca50873 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 8 Jan 2021 15:45:59 -0500
Subject: [PATCH 33/43] Fix expected JSON in html repr test

---
 packages/python/plotly/plotly/tests/test_io/test_renderers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/python/plotly/plotly/tests/test_io/test_renderers.py b/packages/python/plotly/plotly/tests/test_io/test_renderers.py
index 3511201c7d5..27e1e6ea6de 100644
--- a/packages/python/plotly/plotly/tests/test_io/test_renderers.py
+++ b/packages/python/plotly/plotly/tests/test_io/test_renderers.py
@@ -312,7 +312,7 @@ def test_repr_html(renderer):
         "                                    window.PLOTLYENV=window.PLOTLYENV || {};"
         '                                    if (document.getElementById("cd462b94-79ce-42a2-887f-2650a761a144"))'
         ' {                    Plotly.newPlot(                        "cd462b94-79ce-42a2-887f-2650a761a144",'
-        '                        [],                        {"template": {}},'
+        '                        [],                        {"template":{}},'
         '                        {"responsive": true}                    )                };'
         "                            </script>        </div>"
     )

From 76cc62553395215c4d8d967465aa50162a6aad49 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Wed, 13 Jan 2021 10:55:28 -0500
Subject: [PATCH 34/43] Don't drop timezones during serialization, just let
 Plotly.js ignore them

---
 packages/python/plotly/plotly/io/_json.py              |  5 ++---
 .../plotly/tests/test_io/test_to_from_plotly_json.py   | 10 +---------
 2 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 42e80e425fc..08647e6fc41 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -496,9 +496,9 @@ def clean_to_json_compatible(obj, **kwargs):
                 return np.ascontiguousarray(obj.values)
             elif obj.dtype.kind == "M":
                 if isinstance(obj, pd.Series):
-                    dt_values = obj.dt.tz_localize(None).dt.to_pydatetime().tolist()
+                    dt_values = obj.dt.to_pydatetime().tolist()
                 else:  # DatetimeIndex
-                    dt_values = obj.tz_localize(None).to_pydatetime().tolist()
+                    dt_values = obj.to_pydatetime().tolist()
 
                 if not datetime_allowed:
                     # Note: We don't need to handle dropping timezones here because
@@ -513,7 +513,6 @@ def clean_to_json_compatible(obj, **kwargs):
     try:
         # Need to drop timezone for scalar datetimes. Don't need to convert
         # to string since engine can do that
-        obj = obj.replace(tzinfo=None)
         obj = obj.to_pydatetime()
     except (TypeError, AttributeError):
         pass
diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
index a72e64f66e9..1b149d7ba49 100644
--- a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
+++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
@@ -32,7 +32,7 @@ def isoformat_test(dt_value):
     if isinstance(dt_value, np.datetime64):
         return str(dt_value)
     elif isinstance(dt_value, datetime.datetime):
-        return dt_value.replace(tzinfo=None).isoformat()
+        return dt_value.isoformat()
     else:
         raise ValueError("Unsupported date type: {}".format(type(dt_value)))
 
@@ -181,18 +181,10 @@ def test_datetime(datetime_value, engine, pretty):
 
 
 def test_datetime_arrays(datetime_array, engine, pretty):
-    if engine == "legacy":
-        pytest.skip("legacy encoder doesn't strip timezone from datetimes arrays")
-
     value = build_test_dict(datetime_array)
     result = pio.to_json_plotly(value, engine=engine)
 
     def to_str(v):
-        try:
-            v = v.replace(tzinfo=None)
-        except (TypeError, AttributeError):
-            pass
-
         try:
             v = v.isoformat(sep="T")
         except (TypeError, AttributeError):

From 84ba4b556de148ab0eedc225b05792c652ef5d9d Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Wed, 13 Jan 2021 13:53:56 -0500
Subject: [PATCH 35/43] no need to skip legacy tests now

---
 .../plotly/plotly/tests/test_io/test_to_from_plotly_json.py    | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
index 1b149d7ba49..c07a43d23a5 100644
--- a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
+++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
@@ -170,9 +170,6 @@ def test_object_numpy_encoding(object_numpy_array, engine, pretty):
 
 
 def test_datetime(datetime_value, engine, pretty):
-    if engine == "legacy":
-        pytest.skip("legacy encoder doesn't strip timezone from scalar datetimes")
-
     value = build_test_dict(datetime_value)
     result = pio.to_json_plotly(value, engine=engine, pretty=pretty)
     expected = build_test_dict_string('"{}"'.format(isoformat_test(datetime_value)))

From 340aed33bf116530a574badde64e867175c067ad Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Wed, 13 Jan 2021 17:43:24 -0500
Subject: [PATCH 36/43] Only try `datetime_as_string` on datetime kinded numpy
 arrays

---
 packages/python/plotly/_plotly_utils/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/python/plotly/_plotly_utils/utils.py b/packages/python/plotly/_plotly_utils/utils.py
index 572f612793e..9254b7f23d2 100644
--- a/packages/python/plotly/_plotly_utils/utils.py
+++ b/packages/python/plotly/_plotly_utils/utils.py
@@ -186,7 +186,7 @@ def encode_as_numpy(obj):
 
         if obj is numpy.ma.core.masked:
             return float("nan")
-        elif isinstance(obj, numpy.ndarray):
+        elif isinstance(obj, numpy.ndarray) and obj.dtype.kind == "M":
             try:
                 return numpy.datetime_as_string(obj).tolist()
             except TypeError:

From 6cea61db9280b33d4870ff527be9dcafeed45bd3 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 21 Jan 2021 11:36:22 -0500
Subject: [PATCH 37/43] Don't store object or unicode numpy arrays in figure.
 Coerce to lists

---
 .../plotly/_plotly_utils/basevalidators.py    | 69 +++++++-----------
 .../validators/test_dataarray_validator.py    | 13 +++-
 .../validators/test_enumerated_validator.py   |  4 +-
 .../validators/test_pandas_series_input.py    | 28 ++-----
 .../tests/validators/test_string_validator.py |  7 +-
 .../tests/validators/test_xarray_input.py     |  7 +-
 .../plotly/tests/test_core/test_px/test_px.py |  8 +-
 .../test_core/test_px/test_px_functions.py    |  6 +-
 .../tests/test_core/test_px/test_px_input.py  |  2 +-
 scratch/benchmarks.py                         | 73 +++++++++++++++++++
 10 files changed, 132 insertions(+), 85 deletions(-)
 create mode 100644 scratch/benchmarks.py

diff --git a/packages/python/plotly/_plotly_utils/basevalidators.py b/packages/python/plotly/_plotly_utils/basevalidators.py
index 748f2ff70ed..5f3534899b7 100644
--- a/packages/python/plotly/_plotly_utils/basevalidators.py
+++ b/packages/python/plotly/_plotly_utils/basevalidators.py
@@ -53,7 +53,7 @@ def to_scalar_or_list(v):
         return v
 
 
-def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
+def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
     """
     Convert an array-like value into a read-only numpy array
 
@@ -89,7 +89,7 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
 
     # u: unsigned int, i: signed int, f: float
     numeric_kinds = {"u", "i", "f"}
-    kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object"}
+    kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object", "U": "U"}
 
     # Handle pandas Series and Index objects
     if pd and isinstance(v, (pd.Series, pd.Index)):
@@ -113,18 +113,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
     if not isinstance(v, np.ndarray):
         # v has its own logic on how to convert itself into a numpy array
         if is_numpy_convertable(v):
-            return copy_to_readonly_numpy_array(
+            return copy_to_readonly_numpy_array_or_list(
                 np.array(v), kind=kind, force_numeric=force_numeric
             )
         else:
             # v is not homogenous array
-            v_list = [to_scalar_or_list(e) for e in v]
-
-            # Lookup dtype for requested kind, if any
-            dtype = kind_default_dtypes.get(first_kind, None)
-
-            # construct new array from list
-            new_v = np.array(v_list, order="C", dtype=dtype)
+            return [to_scalar_or_list(e) for e in v]
     elif v.dtype.kind in numeric_kinds:
         # v is a homogenous numeric array
         if kind and v.dtype.kind not in kind:
@@ -135,6 +129,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
         else:
             # Either no kind was requested or requested kind is satisfied
             new_v = np.ascontiguousarray(v.copy())
+    elif v.dtype.kind == "O":
+        if kind:
+            dtype = kind_default_dtypes.get(first_kind, None)
+            return np.array(v, dtype=dtype)
+        else:
+            return v.tolist()
     else:
         # v is a non-numeric homogenous array
         new_v = v.copy()
@@ -149,12 +149,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
     if "U" not in kind:
         # Force non-numeric arrays to have object type
         # --------------------------------------------
-        # Here we make sure that non-numeric arrays have the object
-        # datatype. This works around cases like np.array([1, 2, '3']) where
+        # Here we make sure that non-numeric arrays become lists
+        # This works around cases like np.array([1, 2, '3']) where
         # numpy converts the integers to strings and returns array of dtype
         # '<U21'
         if new_v.dtype.kind not in ["u", "i", "f", "O", "M"]:
-            new_v = np.array(v, dtype="object")
+            return v.tolist()
 
     # Set new array to be read-only
     # -----------------------------
@@ -191,7 +191,7 @@ def is_homogeneous_array(v):
             if v_numpy.shape == ():
                 return False
             else:
-                return True
+                return True # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
     return False
 
 
@@ -393,7 +393,7 @@ def validate_coerce(self, v):
             # Pass None through
             pass
         elif is_homogeneous_array(v):
-            v = copy_to_readonly_numpy_array(v)
+            v = copy_to_readonly_numpy_array_or_list(v)
         elif is_simple_array(v):
             v = to_scalar_or_list(v)
         else:
@@ -598,7 +598,7 @@ def validate_coerce(self, v):
                 self.raise_invalid_elements(invalid_els[:10])
 
             if is_homogeneous_array(v):
-                v = copy_to_readonly_numpy_array(v)
+                v = copy_to_readonly_numpy_array_or_list(v)
             else:
                 v = to_scalar_or_list(v)
         else:
@@ -754,7 +754,7 @@ def validate_coerce(self, v):
         elif self.array_ok and is_homogeneous_array(v):
             np = get_module("numpy")
             try:
-                v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
+                v_array = copy_to_readonly_numpy_array_or_list(v, force_numeric=True)
             except (ValueError, TypeError, OverflowError):
                 self.raise_invalid_val(v)
 
@@ -881,7 +881,7 @@ def validate_coerce(self, v):
             pass
         elif self.array_ok and is_homogeneous_array(v):
             np = get_module("numpy")
-            v_array = copy_to_readonly_numpy_array(
+            v_array = copy_to_readonly_numpy_array_or_list(
                 v, kind=("i", "u"), force_numeric=True
             )
 
@@ -1042,26 +1042,7 @@ def validate_coerce(self, v):
                 if invalid_els:
                     self.raise_invalid_elements(invalid_els)
 
-            if is_homogeneous_array(v):
-                np = get_module("numpy")
-
-                # If not strict, let numpy cast elements to strings
-                v = copy_to_readonly_numpy_array(v, kind="U")
-
-                # Check no_blank
-                if self.no_blank:
-                    invalid_els = v[v == ""][:10].tolist()
-                    if invalid_els:
-                        self.raise_invalid_elements(invalid_els)
-
-                # Check values
-                if self.values:
-                    invalid_inds = np.logical_not(np.isin(v, self.values))
-                    invalid_els = v[invalid_inds][:10].tolist()
-                    if invalid_els:
-                        self.raise_invalid_elements(invalid_els)
-
-            elif is_simple_array(v):
+            if is_simple_array(v) or is_homogeneous_array(v):
                 if not self.strict:
                     v = [StringValidator.to_str_or_unicode_or_none(e) for e in v]
 
@@ -1338,8 +1319,8 @@ def validate_coerce(self, v, should_raise=True):
             # Pass None through
             pass
         elif self.array_ok and is_homogeneous_array(v):
-            v = copy_to_readonly_numpy_array(v)
-            if self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
+            v = copy_to_readonly_numpy_array_or_list(v)
+            if not isinstance(v, list) and self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
                 # Numbers are allowed and we have an array of numbers.
                 # All good
                 pass
@@ -1353,9 +1334,9 @@ def validate_coerce(self, v, should_raise=True):
 
                 # ### Check that elements have valid colors types ###
                 elif self.numbers_allowed() or invalid_els:
-                    v = copy_to_readonly_numpy_array(validated_v, kind="O")
+                    v = copy_to_readonly_numpy_array_or_list(validated_v, kind="O")
                 else:
-                    v = copy_to_readonly_numpy_array(validated_v, kind="U")
+                    v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
         elif self.array_ok and is_simple_array(v):
             validated_v = [self.validate_coerce(e, should_raise=False) for e in v]
 
@@ -1870,7 +1851,7 @@ def validate_coerce(self, v):
                 self.raise_invalid_elements(invalid_els)
 
             if is_homogeneous_array(v):
-                v = copy_to_readonly_numpy_array(validated_v, kind="U")
+                v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
             else:
                 v = to_scalar_or_list(v)
         else:
@@ -1918,7 +1899,7 @@ def validate_coerce(self, v):
             # Pass None through
             pass
         elif self.array_ok and is_homogeneous_array(v):
-            v = copy_to_readonly_numpy_array(v, kind="O")
+            v = copy_to_readonly_numpy_array_or_list(v, kind="O")
         elif self.array_ok and is_simple_array(v):
             v = to_scalar_or_list(v)
         return v
diff --git a/packages/python/plotly/_plotly_utils/tests/validators/test_dataarray_validator.py b/packages/python/plotly/_plotly_utils/tests/validators/test_dataarray_validator.py
index fb85863a11d..02beff7f5eb 100644
--- a/packages/python/plotly/_plotly_utils/tests/validators/test_dataarray_validator.py
+++ b/packages/python/plotly/_plotly_utils/tests/validators/test_dataarray_validator.py
@@ -33,7 +33,7 @@ def test_validator_acceptance_simple(val, validator):
 
 @pytest.mark.parametrize(
     "val",
-    [np.array([2, 3, 4]), pd.Series(["a", "b", "c"]), np.array([[1, 2, 3], [4, 5, 6]])],
+    [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
 )
 def test_validator_acceptance_homogeneous(val, validator):
     coerce_val = validator.validate_coerce(val)
@@ -41,6 +41,17 @@ def test_validator_acceptance_homogeneous(val, validator):
     assert np.array_equal(validator.present(coerce_val), val)
 
 
+# Accept object array as list
+@pytest.mark.parametrize(
+    "val",
+    [["A", "B", "C"], np.array(["A", "B", "C"], dtype="object"), pd.Series(["a", "b", "c"])]
+)
+def test_validator_accept_object_array_as_list(val, validator):
+    coerce_val = validator.validate_coerce(val)
+    assert isinstance(coerce_val, list)
+    assert coerce_val == list(val)
+
+
 # ### Rejection ###
 @pytest.mark.parametrize("val", ["Hello", 23, set(), {}])
 def test_rejection(val, validator):
diff --git a/packages/python/plotly/_plotly_utils/tests/validators/test_enumerated_validator.py b/packages/python/plotly/_plotly_utils/tests/validators/test_enumerated_validator.py
index d4daf407018..ec39169b724 100644
--- a/packages/python/plotly/_plotly_utils/tests/validators/test_enumerated_validator.py
+++ b/packages/python/plotly/_plotly_utils/tests/validators/test_enumerated_validator.py
@@ -126,7 +126,7 @@ def test_rejection_by_element_aok(val, validator_aok):
         [],
         ["bar12"],
         ("foo", "bar012", "baz"),
-        np.array([]),
+        np.array([], dtype="object"),
         np.array(["bar12"]),
         np.array(["foo", "bar012", "baz"]),
     ],
@@ -135,7 +135,7 @@ def test_acceptance_aok(val, validator_aok_re):
     # Values should be accepted and returned unchanged
     coerce_val = validator_aok_re.validate_coerce(val)
     if isinstance(val, (np.ndarray, pd.Series)):
-        assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
+        assert coerce_val == list(np.array(val))
     elif isinstance(val, (list, tuple)):
         assert validator_aok_re.present(coerce_val) == tuple(val)
     else:
diff --git a/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py b/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py
index 53e05cd7d09..55a4560f57f 100644
--- a/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py
+++ b/packages/python/plotly/_plotly_utils/tests/validators/test_pandas_series_input.py
@@ -149,13 +149,10 @@ def test_color_validator_object(color_validator, color_object_pandas):
     res = color_validator.validate_coerce(color_object_pandas)
 
     # Check type
-    assert isinstance(res, np.ndarray)
-
-    # Check dtype
-    assert res.dtype == "object"
+    assert isinstance(res, list)
 
     # Check values
-    np.testing.assert_array_equal(res, color_object_pandas)
+    assert res == color_object_pandas.tolist()
 
 
 def test_color_validator_categorical(color_validator, color_categorical_pandas):
@@ -164,13 +161,10 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):
 
     # Check type
     assert color_categorical_pandas.dtype == "category"
-    assert isinstance(res, np.ndarray)
-
-    # Check dtype
-    assert res.dtype == "object"
+    assert isinstance(res, list)
 
     # Check values
-    np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
+    assert res == color_categorical_pandas.tolist()
 
 
 def test_data_array_validator_dates_series(
@@ -180,13 +174,10 @@ def test_data_array_validator_dates_series(
     res = data_array_validator.validate_coerce(datetime_pandas)
 
     # Check type
-    assert isinstance(res, np.ndarray)
-
-    # Check dtype
-    assert res.dtype == "object"
+    assert isinstance(res, list)
 
     # Check values
-    np.testing.assert_array_equal(res, dates_array)
+    assert res == dates_array.tolist()
 
 
 def test_data_array_validator_dates_dataframe(
@@ -197,10 +188,7 @@ def test_data_array_validator_dates_dataframe(
     res = data_array_validator.validate_coerce(df)
 
     # Check type
-    assert isinstance(res, np.ndarray)
-
-    # Check dtype
-    assert res.dtype == "object"
+    assert isinstance(res, list)
 
     # Check values
-    np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))
+    assert res == dates_array.reshape(len(dates_array), 1).tolist()
diff --git a/packages/python/plotly/_plotly_utils/tests/validators/test_string_validator.py b/packages/python/plotly/_plotly_utils/tests/validators/test_string_validator.py
index 957b84c4361..c5aed838e4e 100644
--- a/packages/python/plotly/_plotly_utils/tests/validators/test_string_validator.py
+++ b/packages/python/plotly/_plotly_utils/tests/validators/test_string_validator.py
@@ -138,8 +138,7 @@ def test_acceptance_aok_scalars(val, validator_aok):
 def test_acceptance_aok_list(val, validator_aok):
     coerce_val = validator_aok.validate_coerce(val)
     if isinstance(val, np.ndarray):
-        assert isinstance(coerce_val, np.ndarray)
-        assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
+        assert coerce_val == val.tolist()
     elif isinstance(val, list):
         assert validator_aok.present(val) == tuple(val)
     else:
@@ -178,9 +177,7 @@ def test_rejection_aok_values(val, validator_aok_values):
 )
 def test_acceptance_no_blanks_aok(val, validator_no_blanks_aok):
     coerce_val = validator_no_blanks_aok.validate_coerce(val)
-    if isinstance(val, np.ndarray):
-        assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
-    elif isinstance(val, list):
+    if isinstance(val, (list, np.ndarray)):
         assert validator_no_blanks_aok.present(coerce_val) == tuple(val)
     else:
         assert coerce_val == val
diff --git a/packages/python/plotly/_plotly_utils/tests/validators/test_xarray_input.py b/packages/python/plotly/_plotly_utils/tests/validators/test_xarray_input.py
index b5a885a9238..2e1dca69a53 100644
--- a/packages/python/plotly/_plotly_utils/tests/validators/test_xarray_input.py
+++ b/packages/python/plotly/_plotly_utils/tests/validators/test_xarray_input.py
@@ -126,10 +126,7 @@ def test_color_validator_object(color_validator, color_object_xarray):
     res = color_validator.validate_coerce(color_object_xarray)
 
     # Check type
-    assert isinstance(res, np.ndarray)
-
-    # Check dtype
-    assert res.dtype == "object"
+    assert isinstance(res, list)
 
     # Check values
-    np.testing.assert_array_equal(res, color_object_xarray)
+    assert res == list(color_object_xarray)
diff --git a/packages/python/plotly/plotly/tests/test_core/test_px/test_px.py b/packages/python/plotly/plotly/tests/test_core/test_px/test_px.py
index c3236de32ce..03f10722794 100644
--- a/packages/python/plotly/plotly/tests/test_core/test_px/test_px.py
+++ b/packages/python/plotly/plotly/tests/test_core/test_px/test_px.py
@@ -37,8 +37,8 @@ def test_custom_data_scatter():
         hover_data=["petal_length", "petal_width"],
         custom_data=["species_id", "species"],
     )
-    assert np.all(fig.data[0].customdata[:, 0] == iris.species_id)
-    assert fig.data[0].customdata.shape[1] == 4
+    assert [e[0] for e in fig.data[0].customdata] == iris.species_id.to_list()
+    assert len(fig.data[0].customdata[0]) == 4
     # Hover and custom data, with repeated arguments
     fig = px.scatter(
         iris,
@@ -47,8 +47,8 @@ def test_custom_data_scatter():
         hover_data=["petal_length", "petal_width", "species_id"],
         custom_data=["species_id", "species"],
     )
-    assert np.all(fig.data[0].customdata[:, 0] == iris.species_id)
-    assert fig.data[0].customdata.shape[1] == 4
+    assert [e[0] for e in fig.data[0].customdata] == iris.species_id.tolist()
+    assert len(fig.data[0].customdata[0]) == 4
     assert (
         fig.data[0].hovertemplate
         == "sepal_width=%{x}<br>sepal_length=%{y}<br>petal_length=%{customdata[2]}<br>petal_width=%{customdata[3]}<br>species_id=%{customdata[0]}<extra></extra>"
diff --git a/packages/python/plotly/plotly/tests/test_core/test_px/test_px_functions.py b/packages/python/plotly/plotly/tests/test_core/test_px/test_px_functions.py
index a75a45f43d5..fad79b5ded4 100644
--- a/packages/python/plotly/plotly/tests/test_core/test_px/test_px_functions.py
+++ b/packages/python/plotly/plotly/tests/test_core/test_px/test_px_functions.py
@@ -229,9 +229,9 @@ def test_sunburst_treemap_with_path_color():
     df["hover"] = [el.lower() for el in vendors]
     fig = px.sunburst(df, path=path, color="calls", hover_data=["hover"])
     custom = fig.data[0].customdata
-    assert np.all(custom[:8, 0] == df["hover"])
-    assert np.all(custom[8:, 0] == "(?)")
-    assert np.all(custom[:8, 1] == df["calls"])
+    assert [el[0] for el in custom[:8]] == df["hover"].tolist()
+    assert [el[0] for el in custom[8:]] == ["(?)"] * 7
+    assert [el[1] for el in custom[:8]] == df["calls"].tolist()
 
     # Discrete color
     fig = px.sunburst(df, path=path, color="vendors")
diff --git a/packages/python/plotly/plotly/tests/test_core/test_px/test_px_input.py b/packages/python/plotly/plotly/tests/test_core/test_px/test_px_input.py
index 477e7dbcb04..0dce0ae663e 100644
--- a/packages/python/plotly/plotly/tests/test_core/test_px/test_px_input.py
+++ b/packages/python/plotly/plotly/tests/test_core/test_px/test_px_input.py
@@ -126,7 +126,7 @@ def test_repeated_name():
         hover_data=["petal_length", "petal_width", "species_id"],
         custom_data=["species_id", "species"],
     )
-    assert fig.data[0].customdata.shape[1] == 4
+    assert len(fig.data[0].customdata[0]) == 4
 
 
 def test_arrayattrable_numpy():
diff --git a/scratch/benchmarks.py b/scratch/benchmarks.py
new file mode 100644
index 00000000000..9de88c4afbb
--- /dev/null
+++ b/scratch/benchmarks.py
@@ -0,0 +1,73 @@
+import pickle
+
+from _plotly_utils.optional_imports import get_module
+from plotly.io._json import clean_to_json_compatible
+from plotly.io.json import to_json_plotly
+import time
+import copy
+
+modules = {
+    "sage_all": get_module("sage.all", should_load=False),
+    "np": get_module("numpy", should_load=False),
+    "pd": get_module("pandas", should_load=False),
+    "image": get_module("PIL.Image", should_load=False),
+}
+
+# path = '/home/jmmease/PyDev/repos/plotly.py/doc/timing/json_object/ed9498c3-25ca-4634-82d2-ee8c1837f24a.pkl'
+path = '/doc/timing/json_object0/df55a585-3c9b-4097-8232-6778ecd620a2.pkl'
+with open(path, "rb") as f:
+    json_object = pickle.load(f)
+
+
+def json_clean_json(plotly_object):
+    return clean_to_json_compatible(
+        plotly_object,
+        numpy_allowed=False,
+        datetime_allowed=False,
+        modules=modules,
+    )
+
+
+def json_clean_orjson(plotly_object):
+    return clean_to_json_compatible(
+        plotly_object,
+        numpy_allowed=True,
+        datetime_allowed=True,
+        modules=modules,
+    )
+
+if __name__ == "__main__":
+    trials = 10
+    for engine in ["legacy", "json", "orjson"]:
+        t0 = time.time_ns()
+        for _ in range(trials):
+            to_json_plotly(json_object, engine=engine)
+
+        t = (time.time_ns() - t0) / trials / 1000000
+        print(f"Time for {trials} trials with engine {engine}: {t}")
+
+    clean_json = json_clean_orjson(plotly_object=json_object)
+
+    t0 = time.time_ns()
+    for _ in range(trials):
+        to_json_plotly(clean_json, engine="orjson")
+
+    t = (time.time_ns() - t0) / trials / 1000000
+    print(f"Time for {trials} trials with orjson after cleaning: {t}")
+
+    # # cloning times
+    # t0 = time.time_ns()
+    # for _ in range(trials):
+    #     copy.deepcopy(json_object)
+    #
+    # t = (time.time_ns() - t0) / trials / 1000000
+    # print(f"Time to deep copy for {trials} trials: {t}")
+    #
+    # # Cleaning times
+    # for clean in [json_clean_json, json_clean_orjson]:
+    #     t0 = time.time_ns()
+    #     for _ in range(trials):
+    #         clean(json_object)
+    #
+    #     t = (time.time_ns() - t0) / trials / 1000000
+    #     print(f"Time to clean for {trials} trials with {json_clean_json.__name__}: {t}")

From 93815c147a98e7826a6acd49d90a74ce339cc782 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 21 Jan 2021 12:16:42 -0500
Subject: [PATCH 38/43] Try orjson encoding without cleaning first

---
 packages/python/plotly/plotly/io/_json.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 08647e6fc41..60c330091d6 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -155,6 +155,18 @@ def to_json_plotly(plotly_object, pretty=False, engine=None):
         if pretty:
             opts |= orjson.OPT_INDENT_2
 
+        # Plotly
+        try:
+            plotly_object = plotly_object.to_plotly_json()
+        except AttributeError:
+            pass
+
+        # Try without cleaning
+        try:
+            return orjson.dumps(plotly_object, option=opts).decode("utf8")
+        except TypeError:
+            pass
+
         cleaned = clean_to_json_compatible(
             plotly_object, numpy_allowed=True, datetime_allowed=True, modules=modules,
         )

From 8a3a4b3180e09546f2689b45af704341ea1589ed Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 21 Jan 2021 15:05:18 -0500
Subject: [PATCH 39/43] blacken

---
 .../plotly/_plotly_utils/basevalidators.py       | 16 +++++++++++++---
 .../tests/validators/test_dataarray_validator.py |  9 ++++++---
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/packages/python/plotly/_plotly_utils/basevalidators.py b/packages/python/plotly/_plotly_utils/basevalidators.py
index 5f3534899b7..1dc3ea48677 100644
--- a/packages/python/plotly/_plotly_utils/basevalidators.py
+++ b/packages/python/plotly/_plotly_utils/basevalidators.py
@@ -89,7 +89,13 @@ def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
 
     # u: unsigned int, i: signed int, f: float
     numeric_kinds = {"u", "i", "f"}
-    kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object", "U": "U"}
+    kind_default_dtypes = {
+        "u": "uint32",
+        "i": "int32",
+        "f": "float64",
+        "O": "object",
+        "U": "U",
+    }
 
     # Handle pandas Series and Index objects
     if pd and isinstance(v, (pd.Series, pd.Index)):
@@ -191,7 +197,7 @@ def is_homogeneous_array(v):
             if v_numpy.shape == ():
                 return False
             else:
-                return True # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
+                return True  # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
     return False
 
 
@@ -1320,7 +1326,11 @@ def validate_coerce(self, v, should_raise=True):
             pass
         elif self.array_ok and is_homogeneous_array(v):
             v = copy_to_readonly_numpy_array_or_list(v)
-            if not isinstance(v, list) and self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
+            if (
+                not isinstance(v, list)
+                and self.numbers_allowed()
+                and v.dtype.kind in ["u", "i", "f"]
+            ):
                 # Numbers are allowed and we have an array of numbers.
                 # All good
                 pass
diff --git a/packages/python/plotly/_plotly_utils/tests/validators/test_dataarray_validator.py b/packages/python/plotly/_plotly_utils/tests/validators/test_dataarray_validator.py
index 02beff7f5eb..c5b34a35696 100644
--- a/packages/python/plotly/_plotly_utils/tests/validators/test_dataarray_validator.py
+++ b/packages/python/plotly/_plotly_utils/tests/validators/test_dataarray_validator.py
@@ -32,8 +32,7 @@ def test_validator_acceptance_simple(val, validator):
 
 
 @pytest.mark.parametrize(
-    "val",
-    [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
+    "val", [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
 )
 def test_validator_acceptance_homogeneous(val, validator):
     coerce_val = validator.validate_coerce(val)
@@ -44,7 +43,11 @@ def test_validator_acceptance_homogeneous(val, validator):
 # Accept object array as list
 @pytest.mark.parametrize(
     "val",
-    [["A", "B", "C"], np.array(["A", "B", "C"], dtype="object"), pd.Series(["a", "b", "c"])]
+    [
+        ["A", "B", "C"],
+        np.array(["A", "B", "C"], dtype="object"),
+        pd.Series(["a", "b", "c"]),
+    ],
 )
 def test_validator_accept_object_array_as_list(val, validator):
     coerce_val = validator.validate_coerce(val)

From 1de750af6fbe86e1aff5b24a6c34ed886eb8ddb6 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 21 Jan 2021 15:16:29 -0500
Subject: [PATCH 40/43] remove scratch file

---
 scratch/benchmarks.py | 73 -------------------------------------------
 1 file changed, 73 deletions(-)
 delete mode 100644 scratch/benchmarks.py

diff --git a/scratch/benchmarks.py b/scratch/benchmarks.py
deleted file mode 100644
index 9de88c4afbb..00000000000
--- a/scratch/benchmarks.py
+++ /dev/null
@@ -1,73 +0,0 @@
-import pickle
-
-from _plotly_utils.optional_imports import get_module
-from plotly.io._json import clean_to_json_compatible
-from plotly.io.json import to_json_plotly
-import time
-import copy
-
-modules = {
-    "sage_all": get_module("sage.all", should_load=False),
-    "np": get_module("numpy", should_load=False),
-    "pd": get_module("pandas", should_load=False),
-    "image": get_module("PIL.Image", should_load=False),
-}
-
-# path = '/home/jmmease/PyDev/repos/plotly.py/doc/timing/json_object/ed9498c3-25ca-4634-82d2-ee8c1837f24a.pkl'
-path = '/doc/timing/json_object0/df55a585-3c9b-4097-8232-6778ecd620a2.pkl'
-with open(path, "rb") as f:
-    json_object = pickle.load(f)
-
-
-def json_clean_json(plotly_object):
-    return clean_to_json_compatible(
-        plotly_object,
-        numpy_allowed=False,
-        datetime_allowed=False,
-        modules=modules,
-    )
-
-
-def json_clean_orjson(plotly_object):
-    return clean_to_json_compatible(
-        plotly_object,
-        numpy_allowed=True,
-        datetime_allowed=True,
-        modules=modules,
-    )
-
-if __name__ == "__main__":
-    trials = 10
-    for engine in ["legacy", "json", "orjson"]:
-        t0 = time.time_ns()
-        for _ in range(trials):
-            to_json_plotly(json_object, engine=engine)
-
-        t = (time.time_ns() - t0) / trials / 1000000
-        print(f"Time for {trials} trials with engine {engine}: {t}")
-
-    clean_json = json_clean_orjson(plotly_object=json_object)
-
-    t0 = time.time_ns()
-    for _ in range(trials):
-        to_json_plotly(clean_json, engine="orjson")
-
-    t = (time.time_ns() - t0) / trials / 1000000
-    print(f"Time for {trials} trials with orjson after cleaning: {t}")
-
-    # # cloning times
-    # t0 = time.time_ns()
-    # for _ in range(trials):
-    #     copy.deepcopy(json_object)
-    #
-    # t = (time.time_ns() - t0) / trials / 1000000
-    # print(f"Time to deep copy for {trials} trials: {t}")
-    #
-    # # Cleaning times
-    # for clean in [json_clean_json, json_clean_orjson]:
-    #     t0 = time.time_ns()
-    #     for _ in range(trials):
-    #         clean(json_object)
-    #
-    #     t = (time.time_ns() - t0) / trials / 1000000
-    #     print(f"Time to clean for {trials} trials with {json_clean_json.__name__}: {t}")

From 81f73d50352b4009513db84b93e9a57b05b5bcd0 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Thu, 21 Jan 2021 15:50:25 -0500
Subject: [PATCH 41/43] Remove unused clone

---
 .../python/plotly/plotly/basedatatypes.py     | 28 ++++++-------------
 packages/python/plotly/plotly/io/_utils.py    |  4 +--
 2 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/packages/python/plotly/plotly/basedatatypes.py b/packages/python/plotly/plotly/basedatatypes.py
index 8bfba5e9486..2e50ca8ba23 100644
--- a/packages/python/plotly/plotly/basedatatypes.py
+++ b/packages/python/plotly/plotly/basedatatypes.py
@@ -3273,7 +3273,7 @@ def _perform_batch_animate(self, animation_opts):
 
     # Exports
     # -------
-    def to_dict(self, clone=True):
+    def to_dict(self):
         """
         Convert figure to a dictionary
 
@@ -3286,33 +3286,24 @@ def to_dict(self, clone=True):
         """
         # Handle data
         # -----------
-        if clone:
-            data = deepcopy(self._data)
-        else:
-            data = self._data
+        data = deepcopy(self._data)
 
         # Handle layout
         # -------------
-        if clone:
-            layout = deepcopy(self._layout)
-        else:
-            layout = self._layout
+        layout = deepcopy(self._layout)
 
         # Handle frames
         # -------------
         # Frame key is only added if there are any frames
         res = {"data": data, "layout": layout}
-        if clone:
-            frames = deepcopy([frame._props for frame in self._frame_objs])
-        else:
-            frames = [frame._props for frame in self._frame_objs]
+        frames = deepcopy([frame._props for frame in self._frame_objs])
 
         if frames:
             res["frames"] = frames
 
         return res
 
-    def to_plotly_json(self, clone=True):
+    def to_plotly_json(self):
         """
         Convert figure to a JSON representation as a Python dict
 
@@ -3320,7 +3311,7 @@ def to_plotly_json(self, clone=True):
         -------
         dict
         """
-        return self.to_dict(clone=clone)
+        return self.to_dict()
 
     @staticmethod
     def _to_ordered_dict(d, skip_uid=False):
@@ -5550,7 +5541,7 @@ def on_change(self, callback, *args, **kwargs):
         # -----------------
         self._change_callbacks[arg_tuples].append(callback)
 
-    def to_plotly_json(self, clone=True):
+    def to_plotly_json(self):
         """
         Return plotly JSON representation of object as a Python dict
 
@@ -5558,10 +5549,7 @@ def to_plotly_json(self, clone=True):
         -------
         dict
         """
-        if clone:
-            return deepcopy(self._props if self._props is not None else {})
-        else:
-            return self._props if self._props is not None else {}
+        return deepcopy(self._props if self._props is not None else {})
 
     @staticmethod
     def _vals_equal(v1, v2):
diff --git a/packages/python/plotly/plotly/io/_utils.py b/packages/python/plotly/plotly/io/_utils.py
index 6b5f8b40f81..289ffa2c1a7 100644
--- a/packages/python/plotly/plotly/io/_utils.py
+++ b/packages/python/plotly/plotly/io/_utils.py
@@ -4,11 +4,11 @@
 import plotly.graph_objs as go
 
 
-def validate_coerce_fig_to_dict(fig, validate, clone=True):
+def validate_coerce_fig_to_dict(fig, validate):
     from plotly.basedatatypes import BaseFigure
 
     if isinstance(fig, BaseFigure):
-        fig_dict = fig.to_dict(clone=clone)
+        fig_dict = fig.to_dict()
     elif isinstance(fig, dict):
         if validate:
             # This will raise an exception if fig is not a valid plotly figure

From 80be8bd272d030fdeede5ba57c77f876557cfe66 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 22 Jan 2021 12:38:37 -0500
Subject: [PATCH 42/43] Remove the new "json" encoder since it's sometimes
 slower, and never faster, than current encoder.

Rename "legacy" to "json".
---
 .../python/plotly/plotly/basedatatypes.py     |  6 +--
 packages/python/plotly/plotly/io/_json.py     | 47 ++++---------------
 .../tests/test_io/test_to_from_plotly_json.py |  4 +-
 3 files changed, 13 insertions(+), 44 deletions(-)

diff --git a/packages/python/plotly/plotly/basedatatypes.py b/packages/python/plotly/plotly/basedatatypes.py
index 2e50ca8ba23..038b5ded5cd 100644
--- a/packages/python/plotly/plotly/basedatatypes.py
+++ b/packages/python/plotly/plotly/basedatatypes.py
@@ -3416,9 +3416,8 @@ def to_json(self, *args, **kwargs):
 
         engine: str (default None)
             The JSON encoding engine to use. One of:
-              - "json" for a rewritten encoder based on the built-in Python json module
+              - "json" for an encoder based on the built-in Python json module
               - "orjson" for a fast encoder the requires the orjson package
-              - "legacy" for the legacy JSON encoder.
             If not specified, the default encoder is set to the current value of
             plotly.io.json.config.default_encoder.
 
@@ -3480,9 +3479,8 @@ def write_json(self, *args, **kwargs):
 
         engine: str (default None)
             The JSON encoding engine to use. One of:
-              - "json" for a rewritten encoder based on the built-in Python json module
+              - "json" for an encoder based on the built-in Python json module
               - "orjson" for a fast encoder the requires the orjson package
-              - "legacy" for the legacy JSON encoder.
             If not specified, the default encoder is set to the current value of
             plotly.io.json.config.default_encoder.
 
diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 60c330091d6..8fbf7957683 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -13,7 +13,7 @@
 # Orca configuration class
 # ------------------------
 class JsonConfig(object):
-    _valid_engines = ("legacy", "json", "orjson", "auto")
+    _valid_engines = ("json", "orjson", "auto")
 
     def __init__(self):
         self._default_engine = "auto"
@@ -74,7 +74,6 @@ def to_json_plotly(plotly_object, pretty=False, engine=None):
         The JSON encoding engine to use. One of:
           - "json" for an engine based on the built-in Python json module
           - "orjson" for a faster engine that requires the orjson package
-          - "legacy" for the legacy JSON engine.
           - "auto" for the "orjson" engine if available, otherwise "json"
         If not specified, the default engine is set to the current value of
         plotly.io.json.config.default_engine.
@@ -99,7 +98,7 @@ def to_json_plotly(plotly_object, pretty=False, engine=None):
             engine = "orjson"
         else:
             engine = "json"
-    elif engine not in ["orjson", "json", "legacy"]:
+    elif engine not in ["orjson", "json"]:
         raise ValueError("Invalid json engine: %s" % engine)
 
     modules = {
@@ -111,7 +110,7 @@ def to_json_plotly(plotly_object, pretty=False, engine=None):
 
     # Dump to a JSON string and return
     # --------------------------------
-    if engine in ("json", "legacy"):
+    if engine == "json":
         opts = {"sort_keys": True}
         if pretty:
             opts["indent"] = 2
@@ -119,35 +118,9 @@ def to_json_plotly(plotly_object, pretty=False, engine=None):
             # Remove all whitespace
             opts["separators"] = (",", ":")
 
-        if engine == "json":
-            cleaned = clean_to_json_compatible(
-                plotly_object,
-                numpy_allowed=False,
-                datetime_allowed=False,
-                modules=modules,
-            )
-            encoded_o = json.dumps(cleaned, **opts)
-
-            if not ("NaN" in encoded_o or "Infinity" in encoded_o):
-                return encoded_o
-
-            # now:
-            #    1. `loads` to switch Infinity, -Infinity, NaN to None
-            #    2. `dumps` again so you get 'null' instead of extended JSON
-            try:
-                new_o = json.loads(encoded_o, parse_constant=coerce_to_strict)
-            except ValueError:
-                # invalid separators will fail here. raise a helpful exception
-                raise ValueError(
-                    "Encoding into strict JSON failed. Did you set the separators "
-                    "valid JSON separators?"
-                )
-            else:
-                return json.dumps(new_o, **opts)
-        else:
-            from _plotly_utils.utils import PlotlyJSONEncoder
+        from _plotly_utils.utils import PlotlyJSONEncoder
 
-            return json.dumps(plotly_object, cls=PlotlyJSONEncoder, **opts)
+        return json.dumps(plotly_object, cls=PlotlyJSONEncoder, **opts)
     elif engine == "orjson":
         JsonConfig.validate_orjson()
         opts = orjson.OPT_SORT_KEYS | orjson.OPT_SERIALIZE_NUMPY
@@ -197,7 +170,6 @@ def to_json(fig, validate=True, pretty=False, remove_uids=True, engine=None):
         The JSON encoding engine to use. One of:
           - "json" for an engine based on the built-in Python json module
           - "orjson" for a faster engine that requires the orjson package
-          - "legacy" for the legacy JSON engine.
           - "auto" for the "orjson" engine if available, otherwise "json"
         If not specified, the default engine is set to the current value of
         plotly.io.json.config.default_engine.
@@ -249,7 +221,6 @@ def write_json(fig, file, validate=True, pretty=False, remove_uids=True, engine=
         The JSON encoding engine to use. One of:
           - "json" for an engine based on the built-in Python json module
           - "orjson" for a faster engine that requires the orjson package
-          - "legacy" for the legacy JSON engine.
           - "auto" for the "orjson" engine if available, otherwise "json"
         If not specified, the default engine is set to the current value of
         plotly.io.json.config.default_engine.
@@ -289,7 +260,7 @@ def from_json_plotly(value, engine=None):
 
     engine: str (default None)
         The JSON decoding engine to use. One of:
-          - if "json" or "legacy", parse JSON using built in json module
+          - if "json", parse JSON using built in json module
           - if "orjson", parse using the faster orjson module, requires the orjson
             package
           - if "auto" use orjson module if available, otherwise use the json module
@@ -327,7 +298,7 @@ def from_json_plotly(value, engine=None):
             engine = "orjson"
         else:
             engine = "json"
-    elif engine not in ["orjson", "json", "legacy"]:
+    elif engine not in ["orjson", "json"]:
         raise ValueError("Invalid json engine: %s" % engine)
 
     if engine == "orjson":
@@ -362,7 +333,7 @@ def from_json(value, output_type="Figure", skip_invalid=False, engine=None):
 
     engine: str (default None)
         The JSON decoding engine to use. One of:
-          - if "json" or "legacy", parse JSON using built in json module
+          - if "json", parse JSON using built in json module
           - if "orjson", parse using the faster orjson module, requires the orjson
             package
           - if "auto" use orjson module if available, otherwise use the json module
@@ -416,7 +387,7 @@ def read_json(file, output_type="Figure", skip_invalid=False, engine=None):
 
     engine: str (default None)
         The JSON decoding engine to use. One of:
-          - if "json" or "legacy", parse JSON using built in json module
+          - if "json", parse JSON using built in json module
           - if "orjson", parse using the faster orjson module, requires the orjson
             package
           - if "auto" use orjson module if available, otherwise use the json module
diff --git a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
index c07a43d23a5..70d6803f1ac 100644
--- a/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
+++ b/packages/python/plotly/plotly/tests/test_io/test_to_from_plotly_json.py
@@ -65,9 +65,9 @@ def check_roundtrip(value, engine, pretty):
 
 # Fixtures
 if orjson is not None:
-    engines = ["json", "orjson", "legacy", "auto"]
+    engines = ["json", "orjson", "auto"]
 else:
-    engines = ["json", "legacy", "auto"]
+    engines = ["json", "auto"]
 
 
 @pytest.fixture(scope="module", params=engines)

From cb54f8892bce0832f373a255de3d3c52b0822928 Mon Sep 17 00:00:00 2001
From: Jon Mease <jon.mease.dev@gmail.com>
Date: Fri, 22 Jan 2021 12:53:28 -0500
Subject: [PATCH 43/43] Reorder dict cleaning for performance

---
 packages/python/plotly/plotly/io/_json.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/packages/python/plotly/plotly/io/_json.py b/packages/python/plotly/plotly/io/_json.py
index 8fbf7957683..3bda778c76e 100644
--- a/packages/python/plotly/plotly/io/_json.py
+++ b/packages/python/plotly/plotly/io/_json.py
@@ -429,6 +429,13 @@ def clean_to_json_compatible(obj, **kwargs):
     if isinstance(obj, (int, float, string_types)):
         return obj
 
+    if isinstance(obj, dict):
+        return {k: clean_to_json_compatible(v, **kwargs) for k, v in obj.items()}
+    elif isinstance(obj, (list, tuple)):
+        if obj:
+            # Must process list recursively even though it may be slow
+            return [clean_to_json_compatible(v, **kwargs) for v in obj]
+
     # unpack kwargs
     numpy_allowed = kwargs.get("numpy_allowed", False)
     datetime_allowed = kwargs.get("datetime_allowed", False)
@@ -439,12 +446,6 @@ def clean_to_json_compatible(obj, **kwargs):
     pd = modules["pd"]
     image = modules["image"]
 
-    # Plotly
-    try:
-        obj = obj.to_plotly_json()
-    except AttributeError:
-        pass
-
     # Sage
     if sage_all is not None:
         if obj in sage_all.RR:
@@ -522,6 +523,12 @@ def clean_to_json_compatible(obj, **kwargs):
     if image is not None and isinstance(obj, image.Image):
         return ImageUriValidator.pil_image_to_uri(obj)
 
+    # Plotly
+    try:
+        obj = obj.to_plotly_json()
+    except AttributeError:
+        pass
+
     # Recurse into lists and dictionaries
     if isinstance(obj, dict):
         return {k: clean_to_json_compatible(v, **kwargs) for k, v in obj.items()}