Multiplex scalar fetch: one tag, many runs

Author: davidsoergel

tensorboard/components/tf_dashboard_common/data-loader-behavior.ts

@@ -81,13 +81,22 @@ namespace tf_dashboard_common {
       requestData: {
         type: Function,
         value: function() {
-          return (datum) =>
-            this.requestManager.request(this.getDataLoadUrl(datum));
+          return (datum) => {
+            const dataLoadUrl = this.getDataLoadUrl(datum);
+            var url;
+            var postdata = {};
+            if (Array.isArray(dataLoadUrl)) {
+              [url, postdata] = dataLoadUrl;
+            } else {
+              url = dataLoadUrl;
+            }
+            return this.requestManager.request(url, postdata);
+          };
         },
       },
 
       // A function that takes a datum and returns a string URL for fetching
-      // data.
+      // data.  Optionally, returns a tuple of (URL, postdata).
       getDataLoadUrl: Function,
 
       dataLoading: {

tensorboard/plugins/scalar/scalars_plugin.py

@@ -24,6 +24,7 @@
 
 import collections
 import csv
+import json
 
 import six
 from six import StringIO
@@ -95,29 +96,37 @@ def index_impl(self, ctx, experiment=None):
                 }
         return result
 
-    def scalars_impl(self, ctx, tag, run, experiment, output_format):
+    def scalars_impl(self, ctx, tag, runs, experiment, output_format):
         """Result of the form `(body, mime_type)`."""
         all_scalars = self._data_provider.read_scalars(
             ctx,
             experiment_id=experiment,
             plugin_name=metadata.PLUGIN_NAME,
             downsample=self._downsample_to,
-            run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]),
+            run_tag_filter=provider.RunTagFilter(runs=runs, tags=[tag]),
         )
-        scalars = all_scalars.get(run, {}).get(tag, None)
-        if scalars is None:
-            raise errors.NotFoundError(
-                "No scalar data for run=%r, tag=%r" % (run, tag)
-            )
-        values = [(x.wall_time, x.step, x.value) for x in scalars]
+        result = {}
+        for run in all_scalars:
+            scalars = all_scalars.get(run, {}).get(tag, [])
+            # if scalars is None:
+            #     raise errors.NotFoundError(
+            #         "No scalar data for run=%r, tag=%r" % (run, tag)
+            #     )
+            values = [(x.wall_time, x.step, x.value) for x in scalars]
+            result[run] = values
+
         if output_format == OutputFormat.CSV:
+            if(runs.length > 1):
+                raise errors.InvalidArgumentError(
+                    "Not implemented: Return CSV data for more than one run "
+                    "at a time.")
             string_io = StringIO()
             writer = csv.writer(string_io)
             writer.writerow(["Wall time", "Step", "Value"])
             writer.writerows(values)
             return (string_io.getvalue(), "text/csv")
         else:
-            return (values, "application/json")
+            return (json.dumps(result), "application/json")
 
     @wrappers.Request.application
     def tags_route(self, request):
@@ -129,12 +138,31 @@ def tags_route(self, request):
     @wrappers.Request.application
     def scalars_route(self, request):
         """Given a tag and single run, return array of ScalarEvents."""
-        tag = request.args.get("tag")
-        run = request.args.get("run")
+        if request.method == "GET":
+            tag = request.args.get("tag")
+            run = request.args.get("run")
+            runs = [run]
+        else:
+            tag = request.form["tag"]
+            json_runs = request.form["runs"]
+            try:
+                runs = json.loads(json_runs)
+            except Exception as e:  # pylint: disable=broad-except
+                # Different JSON libs raise different exceptions, so we just do a
+                # catch-all here. This problem is complicated by how Tensorboard might be
+                # run in many different environments, as it is open-source.
+                # TODO(@caisq, @chihuahua): Create platform-dependent adapter to catch
+                # specific types of exceptions, instead of the broad catching here.
+                response = ("Could not decode runs JSON string %r: %s") % (
+                    json_runs,
+                    e,
+                )
+                return http_util.Respond(request, response, "text/plain", code=400)
+
         ctx = plugin_util.context(request.environ)
         experiment = plugin_util.experiment_id(request.environ)
         output_format = request.args.get("format")
         (body, mime_type) = self.scalars_impl(
-            ctx, tag, run, experiment, output_format
+            ctx, tag, runs, experiment, output_format
         )
         return http_util.Respond(request, body, mime_type)

tensorboard/plugins/scalar/tf_scalar_dashboard/tf-scalar-card.html

@@ -45,8 +45,8 @@
       <tf-line-chart-data-loader
         active="[[active]]"
         color-scale="[[_getColorScale(colorScale)]]"
-        data-series="[[_getDataSeries(dataToLoad.*)]]"
-        data-to-load="[[dataToLoad]]"
+        data-series="[[_getDataSeries(tagAndRuns.series)]]"
+        data-to-load="[[_asSingletonArray(tagAndRuns)]]"
         get-data-load-name="[[_getDataLoadName]]"
         get-data-load-url="[[getDataLoadUrl]]"
         request-data="[[requestData]]"
@@ -102,7 +102,7 @@
       <template is="dom-if" if="[[showDownloadLinks]]">
         <div class="download-links">
           <tf-downloader
-            runs="[[_runsFromData(dataToLoad)]]"
+            runs="[[_runsFromData(tagAndRuns.series)]]"
             tag="[[tag]]"
             url-fn="[[_downloadUrlFn]]"
           ></tf-downloader>
@@ -191,7 +191,9 @@
         /**
          * @type {Array<Object>}
          */
-        dataToLoad: Array,
+        // dataToLoad: Array,
+
+        tagAndRuns: Object,
 
         /**
          * @type {vz_chart_helpers.XType}
@@ -218,14 +220,17 @@
           type: Object,
           value: function() {
             return (scalarChart, datum, data) => {
-              const formattedData = data.map((datum) => ({
-                wall_time: new Date(datum[0] * 1000),
-                step: datum[1],
-                scalar: datum[2],
-              }));
-              const name = this._getSeriesNameFromDatum(datum);
-              scalarChart.setSeriesMetadata(name, datum);
-              scalarChart.setSeriesData(name, formattedData);
+              for (const run in data) {
+                const points = data[run];
+                const formattedData = points.map((point) => ({
+                  wall_time: new Date(point[0] * 1000),
+                  step: point[1],
+                  scalar: point[2],
+                }));
+                const name = this._getSeriesNameForRun(run, datum.experiment);
+                scalarChart.setSeriesMetadata(name, {run, tag: datum.tag});
+                scalarChart.setSeriesData(name, formattedData);
+              }
               scalarChart.commitChanges();
             };
           },
@@ -235,14 +240,12 @@
         getDataLoadUrl: {
           type: Function,
           value: function() {
-            return ({tag, run}) => {
-              return tf_backend
+            return ({tag, runs}) => {
+              const url = tf_backend
                 .getRouter()
-                .pluginRoute(
-                  'scalars',
-                  '/scalars',
-                  new URLSearchParams({tag, run})
-                );
+                .pluginRoute('scalars', '/scalars');
+              const postdata = {tag, runs: JSON.stringify(runs)};
+              return [url, postdata];
             };
           },
         },
@@ -266,7 +269,7 @@
         _getDataLoadName: {
           type: Function,
           value: function() {
-            return (datum) => this._getSeriesNameFromDatum(datum);
+            return (datum) => datum.tag;
           },
         },
 
@@ -325,10 +328,12 @@
         return data.map((datum) => datum.run);
       },
       _getDataSeries() {
-        return this.dataToLoad.map((d) => this._getSeriesNameFromDatum(d));
+        return this.tagAndRuns.series.map((d) =>
+          this._getSeriesNameForRun(d.run, d.experiment)
+        );
       },
       // name is a stable identifier for a series.
-      _getSeriesNameFromDatum({run, experiment = {name: '_default'}}) {
+      _getSeriesNameForRun(run, experiment = {name: '_default'}) {
         return JSON.stringify([experiment.name, run]);
       },
       // title is a visible string of a series for the UI.
@@ -348,6 +353,9 @@
           },
         };
       },
+      _asSingletonArray(x) {
+        return [x];
+      },
     });
   </script>
 </dom-module>

tensorboard/plugins/scalar/tf_scalar_dashboard/tf-scalar-dashboard.html

@@ -162,7 +162,7 @@ <h3>No scalar data was found.</h3>
               <template>
                 <tf-scalar-card
                   active="[[active]]"
-                  data-to-load="[[item.series]]"
+                  tag-and-runs="[[item]]"
                   ignore-y-outliers="[[_ignoreYOutliers]]"
                   multi-experiments="[[_getMultiExperiments(dataSelection)]]"
                   request-manager="[[_requestManager]]"
@@ -361,6 +361,7 @@ <h3>No scalar data was found.</h3>
         categories.forEach((category) => {
           category.items = category.items.map((item) => ({
             tag: item.tag,
+            runs: item.runs,
             series: item.runs.map((run) => ({run, tag: item.tag})),
           }));
         });