plots: return errors in json format

dvc/commands/plots.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import os
+from typing import TYPE_CHECKING, List
 
 from funcy import first
 
@@ -12,14 +13,24 @@
 from dvc.ui import ui
 from dvc.utils import format_link
 
+if TYPE_CHECKING:
+    from dvc.render.match import RendererWithErrors
+
+
 logger = logging.getLogger(__name__)
 
 
-def _show_json(renderers, split=False):
+def _show_json(renderers: List["RendererWithErrors"], split=False):
     from dvc.render.convert import to_json
 
-    result = {renderer.name: to_json(renderer, split) for renderer in renderers}
-    ui.write_json(result)
+    result = {
+        renderer.name: to_json(renderer, src_errors, def_errors, split)
+        for renderer, src_errors, def_errors in renderers
+    }
+
+    from dvc.utils.serialize import encode_exception
+
+    ui.write_json(result, default=encode_exception)
 
 
 def _adjust_vega_renderers(renderers):
@@ -123,15 +134,16 @@ def run(self) -> int:  # noqa: C901, PLR0911, PLR0912
 
             renderers_out = out if self.args.json else os.path.join(out, "static")
 
-            renderers = match_defs_renderers(
+            renderers_with_errors = match_defs_renderers(
                 data=plots_data,
                 out=renderers_out,
                 templates_dir=self.repo.plots.templates_dir,
             )
             if self.args.json:
-                _show_json(renderers, self.args.split)
+                _show_json(renderers_with_errors, self.args.split)
                 return 0
 
+            renderers = [r.renderer for r in renderers_with_errors]
             _adjust_vega_renderers(renderers)
             if self.args.show_vega:
                 renderer = first(filter(lambda r: r.TYPE == "vega", renderers))

dvc/render/convert.py

@@ -1,6 +1,6 @@
 import json
 from collections import defaultdict
-from typing import Dict, List, Union
+from typing import Union
 
 from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD
 from dvc.render.converter.image import ImageConverter
@@ -28,30 +28,74 @@ def _group_by_rev(datapoints):
     return dict(grouped)
 
 
-def to_json(renderer, split: bool = False) -> List[Dict]:
+def _to_json_image(renderer, rev, datapoint, src_errors, def_error):
+    url = datapoint.get(SRC_FIELD)
+    d = {
+        TYPE_KEY: renderer.TYPE,
+        REVISIONS_KEY: [rev],
+    }
+    if url:
+        d["url"] = url
+
+    errors = {}
+    if def_error is not None:
+        errors["definitions"] = {rev: def_error}
+    if src_errors:
+        errors["sources"] = {rev: src_errors}
+    if errors:
+        d["errors"] = errors
+    return d
+
+
+def _to_json_images(renderer, src_errors, def_errors):
+    d = {}
+
+    for datapoint in renderer.datapoints:
+        rev = datapoint.get(REVISION_FIELD)
+        src_error = src_errors.get(rev, None)
+        def_error = def_errors.get(rev, None)
+        d[rev] = _to_json_image(renderer, rev, datapoint, src_error, def_error)
+
+    # add from revisions where we have no datapoints but only errors
+    for rev in (def_errors.keys() | src_errors.keys()) - d.keys():
+        src_error = src_errors.get(rev, None)
+        def_error = def_errors.get(rev, None)
+        d[rev] = _to_json_image(renderer, rev, {}, src_error, def_error)
+    return list(d.values())
+
+
+def _to_json_vega(renderer, src_errors, def_errors, split: bool = False):
+    grouped = _group_by_rev(renderer.datapoints)
+    if split:
+        content = renderer.get_filled_template(skip_anchors=["data"])
+    else:
+        content = renderer.get_filled_template()
+
+    d = {
+        TYPE_KEY: renderer.TYPE,
+        REVISIONS_KEY: sorted(grouped.keys()),
+    }
+    if grouped:
+        d["datapoints"] = grouped
+    if content:
+        d["content"] = json.loads(content)
+
+    errors = {}
+    if src_errors:
+        errors["sources"] = src_errors
+    if def_errors:
+        errors["definitions"] = def_errors
+    if errors:
+        d["errors"] = errors
+    return [d]
+
+
+def to_json(renderer, src_errors=None, def_errors=None, split: bool = False):
+    def_errors = def_errors or {}
+    src_errors = src_errors or {}
+
     if renderer.TYPE == "vega":
-        grouped = _group_by_rev(renderer.datapoints)
-        if split:
-            content = renderer.get_filled_template(skip_anchors=["data"])
-        else:
-            content = renderer.get_filled_template()
-        if grouped:
-            return [
-                {
-                    TYPE_KEY: renderer.TYPE,
-                    REVISIONS_KEY: sorted(grouped.keys()),
-                    "content": json.loads(content),
-                    "datapoints": grouped,
-                }
-            ]
-        return []
+        return _to_json_vega(renderer, src_errors, def_errors, split=split)
     if renderer.TYPE == "image":
-        return [
-            {
-                TYPE_KEY: renderer.TYPE,
-                REVISIONS_KEY: [datapoint.get(REVISION_FIELD)],
-                "url": datapoint.get(SRC_FIELD),
-            }
-            for datapoint in renderer.datapoints
-        ]
+        return _to_json_images(renderer, src_errors, def_errors)
     raise ValueError(f"Invalid renderer: {renderer.TYPE}")

dvc/render/match.py

@@ -1,6 +1,6 @@
 import os
 from collections import defaultdict
-from typing import TYPE_CHECKING, Dict, List, Optional
+from typing import TYPE_CHECKING, DefaultDict, Dict, List, NamedTuple, Optional
 
 import dpath
 import dpath.options
@@ -13,6 +13,8 @@
 
 if TYPE_CHECKING:
     from dvc.types import StrPath
+    from dvc_render.base import Renderer
+
 
 dpath.options.ALLOW_EMPTY_STRING_KEYS = True
 
@@ -61,26 +63,38 @@ def get_definition_data(self, target_files, rev):
         return result
 
 
-def match_defs_renderers(
+class RendererWithErrors(NamedTuple):
+    renderer: "Renderer"
+    source_errors: Dict[str, Dict[str, Exception]]
+    definition_errors: Dict[str, Exception]
+
+
+def match_defs_renderers(  # noqa: C901, PLR0912
     data,
     out=None,
     templates_dir: Optional["StrPath"] = None,
-):
+) -> List[RendererWithErrors]:
     from dvc_render import ImageRenderer, VegaRenderer
 
     plots_data = PlotsData(data)
     renderers = []
     renderer_cls = None
+
     for plot_id, group in plots_data.group_definitions().items():
         plot_datapoints: List[Dict] = []
         props = _squash_plots_properties(group)
         final_props: Dict = {}
 
+        def_errors: Dict[str, Exception] = {}
+        src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict)
+
         if out is not None:
             props["out"] = out
         if templates_dir is not None:
             props["template_dir"] = templates_dir
 
+        from funcy import get_in
+
         for rev, inner_id, plot_definition in group:
             plot_sources = infer_data_sources(inner_id, plot_definition)
             definitions_data = plots_data.get_definition_data(plot_sources, rev)
@@ -94,13 +108,24 @@ def match_defs_renderers(
 
             converter = _get_converter(renderer_cls, inner_id, props, definitions_data)
 
-            dps, rev_props = converter.flat_datapoints(rev)
+            try:
+                dps, rev_props = converter.flat_datapoints(rev)
+            except Exception as e:  # noqa: BLE001, pylint: disable=broad-except
+                def_errors[rev] = e
+                continue
+
             if not final_props and rev_props:
                 final_props = rev_props
             plot_datapoints.extend(dps)
 
+            for src in plot_sources:
+                if error := get_in(data, [rev, "sources", "data", src, "error"]):
+                    src_errors[rev][src] = error
+
         if "title" not in final_props:
             final_props["title"] = renderer_id
+
         if renderer_cls is not None:
-            renderers.append(renderer_cls(plot_datapoints, renderer_id, **final_props))
+            renderer = renderer_cls(plot_datapoints, renderer_id, **final_props)
+            renderers.append(RendererWithErrors(renderer, dict(src_errors), def_errors))
     return renderers

dvc/repo/plots/__init__.py

@@ -20,7 +20,7 @@
 
 import dpath
 import dpath.options
-from funcy import distinct, first, project
+from funcy import first, ldistinct, project
 
 from dvc.exceptions import DvcException
 from dvc.utils import error_handler, errored_revisions, onerror_collect
@@ -333,7 +333,7 @@ def infer_data_sources(plot_id, config=None):
     if isinstance(x, dict):
         sources.append(first(x.keys()))
 
-    return distinct(source for source in sources)
+    return ldistinct(source for source in sources)
 
 
 def _matches(targets, config_file, plot_id):

tests/integration/plots/test_plots.py

@@ -366,8 +366,22 @@ def test_repo_with_removed_plots(tmp_dir, capsys, repo_with_plots):
             "confusion.json",
             "image.png",
         }:
-            assert json_result[p] == []
-            assert split_json_result[p] == []
+            expected_info = {
+                "type": "image" if p.endswith(".png") else "vega",
+                "revisions": ["workspace"] if p.endswith(".png") else [],
+                "errors": {
+                    "sources": {
+                        "workspace": {
+                            p: {
+                                "msg": "",
+                                "type": "FileNotFoundError",
+                            }
+                        }
+                    }
+                },
+            }
+            assert json_result[p] == [expected_info]
+            assert split_json_result[p] == [expected_info]
 
 
 def test_config_output_dir(tmp_dir, dvc, capsys):

tests/unit/command/test_plots.py

@@ -7,6 +7,7 @@
 
 from dvc.cli import parse_args
 from dvc.commands.plots import CmdPlotsDiff, CmdPlotsShow, CmdPlotsTemplates
+from dvc.render.match import RendererWithErrors
 
 
 @pytest.fixture
@@ -268,8 +269,11 @@ def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output):
 
     output = output or "dvc_plots"
     index_path = tmp_dir / output / "index.html"
-    renderers = mocker.MagicMock()
-    mocker.patch("dvc.render.match.match_defs_renderers", return_value=renderers)
+    renderer = mocker.MagicMock()
+    mocker.patch(
+        "dvc.render.match.match_defs_renderers",
+        return_value=[RendererWithErrors(renderer, {}, {})],
+    )
     render_mock = mocker.patch("dvc_render.render_html", return_value=index_path)
 
     assert cmd.run() == 0
@@ -278,7 +282,7 @@ def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output):
     assert index_path.as_uri() in out
 
     render_mock.assert_called_once_with(
-        renderers=renderers,
+        renderers=[renderer],
         output_file=Path(tmp_dir / output / "index.html"),
         html_template=None,
     )
@@ -354,14 +358,15 @@ def test_show_json(split, mocker, capsys):
     import dvc.commands.plots
 
     renderer = mocker.MagicMock()
+    renderer_obj = RendererWithErrors(renderer, {}, {})
     renderer.name = "rname"
     to_json_mock = mocker.patch(
         "dvc.render.convert.to_json", return_value={"renderer": "json"}
     )
 
-    dvc.commands.plots._show_json([renderer], split)
+    dvc.commands.plots._show_json([renderer_obj], split)
 
-    to_json_mock.assert_called_once_with(renderer, split)
+    to_json_mock.assert_called_once_with(renderer, {}, {}, split)
 
     out, _ = capsys.readouterr()
     assert json.dumps({"rname": {"renderer": "json"}}) in out

tests/unit/render/test_match.py

@@ -77,9 +77,9 @@ def test_match_renderers(mocker):
         },
     }
 
-    renderers = match_defs_renderers(data)
-    assert len(renderers) == 1
-    assert renderers[0].datapoints == [
+    (renderer_with_errors,) = match_defs_renderers(data)
+    renderer = renderer_with_errors[0]
+    assert renderer.datapoints == [
         {
             VERSION_FIELD: {
                 "revision": "v1",
@@ -99,7 +99,7 @@ def test_match_renderers(mocker):
             "y": 2,
         },
     ]
-    assert renderers[0].properties == {
+    assert renderer.properties == {
         "title": "config_file_1::plot_id_1",
         "x": "x",
         "y": "y",