Scalars: Multiplex fetch (one tag, many runs).

tensorboard/components/tf_dashboard_common/data-loader-behavior.ts

@@ -81,13 +81,22 @@ namespace tf_dashboard_common {
       requestData: {
         type: Function,
         value: function() {
-          return (datum) =>
-            this.requestManager.request(this.getDataLoadUrl(datum));
+          return (datum) => {
+            const dataLoadUrl = this.getDataLoadUrl(datum);
+            let url;
+            let postData;
+            if (typeof dataLoadUrl === 'string') {
+              url = dataLoadUrl;
+            } else {
+              ({url, postData} = dataLoadUrl);
+            }
+            return this.requestManager.request(url, postData);
+          };
         },
       },
 
       // A function that takes a datum and returns a string URL for fetching
-      // data.
+      // data.  Optionally, returns an object like {url, postData}.
       getDataLoadUrl: Function,
 
       dataLoading: {

tensorboard/plugins/scalar/http_api.md

@@ -61,3 +61,16 @@ instead be in CSV format:
     1443856985.705543,1448,0.7461960315704346
     1443857105.704628,3438,0.5427092909812927
     1443857225.705133,5417,0.5457325577735901
+
+## `/data/plugin/scalars/scalars_multirun`
+
+Accepts a POST request where the form data has a single field called `query`,
+containing a JSON-encoded dict of this shape:
+
+    {
+      "tag": "foo",
+      "runs": ["bar", "baz"],
+    }
+
+Returns a dict, keyed by run name, where each value is an array of the form
+`[wall_time, step, value]` as above.

tensorboard/plugins/scalar/scalars_plugin.py

@@ -24,6 +24,7 @@
 
 import collections
 import csv
+import json
 
 import six
 from six import StringIO
@@ -69,6 +70,7 @@ def get_plugin_apps(self):
         return {
             "/scalars": self.scalars_route,
             "/tags": self.tags_route,
+            "/scalars_multirun": self.scalars_multirun_route,
         }
 
     def is_active(self):
@@ -119,6 +121,25 @@ def scalars_impl(self, ctx, tag, run, experiment, output_format):
         else:
             return (values, "application/json")
 
+    def _scalars_multirun_impl(self, ctx, tag, runs, experiment):
+        """Result of the form `(body, mime_type)`."""
+        all_scalars = self._data_provider.read_scalars(
+            ctx,
+            experiment_id=experiment,
+            plugin_name=metadata.PLUGIN_NAME,
+            downsample=self._downsample_to,
+            run_tag_filter=provider.RunTagFilter(runs=runs, tags=[tag]),
+        )
+        result = {}
+        # Note we do not raise an error if data for a given run was not
+        # found; we just omit the run that case.
+        for run in all_scalars:
+            scalars = all_scalars[run][tag]
+            values = [(x.wall_time, x.step, x.value) for x in scalars]
+            result[run] = values
+
+        return (result, "application/json")
+
     @wrappers.Request.application
     def tags_route(self, request):
         ctx = plugin_util.context(request.environ)
@@ -131,10 +152,29 @@ def scalars_route(self, request):
         """Given a tag and single run, return array of ScalarEvents."""
         tag = request.args.get("tag")
         run = request.args.get("run")
+
         ctx = plugin_util.context(request.environ)
         experiment = plugin_util.experiment_id(request.environ)
         output_format = request.args.get("format")
         (body, mime_type) = self.scalars_impl(
             ctx, tag, run, experiment, output_format
         )
         return http_util.Respond(request, body, mime_type)
+
+    @wrappers.Request.application
+    def scalars_multirun_route(self, request):
+        """Given a tag and runs, return dict of run to array of ScalarEvents."""
+        try:
+            query = json.loads(request.form["query"])
+        except json.JSONDecodeError as e:
+            raise errors.InvalidArgumentError(e)
+
+        tag = query["tag"]
+        runs = query["runs"]
+
+        ctx = plugin_util.context(request.environ)
+        experiment = plugin_util.experiment_id(request.environ)
+        (body, mime_type) = self._scalars_multirun_impl(
+            ctx, tag, runs, experiment
+        )
+        return http_util.Respond(request, body, mime_type)

tensorboard/plugins/scalar/scalars_plugin_test.py

@@ -66,6 +66,7 @@ class ScalarsPluginTest(tf.test.TestCase):
     _RUN_WITH_LEGACY_SCALARS = "_RUN_WITH_LEGACY_SCALARS"
     _RUN_WITH_SCALARS = "_RUN_WITH_SCALARS"
     _RUN_WITH_HISTOGRAM = "_RUN_WITH_HISTOGRAM"
+    _RUN_WITH_NOTHING = "_RUN_WITH_NOTHING"
 
     def load_plugin(self, run_names):
         logdir = self.get_temp_dir()
@@ -110,6 +111,8 @@ def generate_run(self, logdir, run_name):
                     summ = tf.compat.v1.summary.histogram(
                         self._HISTOGRAM_TAG, data
                     ).numpy()
+                elif run_name == self._RUN_WITH_NOTHING:
+                    continue
                 else:
                     assert False, "Invalid run name: %r" % run_name
                 writer.add_summary(summ, global_step=step)
@@ -171,6 +174,66 @@ def test_scalars_with_scalars(self):
         self.assertEqual("application/json", response.headers["Content-Type"])
         self.assertEqual(self._STEPS, len(json.loads(response.get_data())))
 
+    def test_scalars_multirun_with_scalars(self):
+        server = self.load_server(
+            [self._RUN_WITH_SCALARS, self._RUN_WITH_HISTOGRAM]
+        )
+        response = server.post(
+            "/data/plugin/scalars/scalars_multirun",
+            data={
+                "query": json.dumps(
+                    {
+                        "tag": "%s/scalar_summary" % self._SCALAR_TAG,
+                        "runs": [
+                            self._RUN_WITH_SCALARS,
+                            self._RUN_WITH_HISTOGRAM,
+                        ],
+                    }
+                )
+            },
+        )
+        self.assertEqual(200, response.status_code)
+        self.assertEqual("application/json", response.headers["Content-Type"])
+        r = json.loads(response.get_data())
+        self.assertItemsEqual([self._RUN_WITH_SCALARS], r.keys())
+        self.assertEqual(self._STEPS, len(r[self._RUN_WITH_SCALARS]))
+
+    def test_scalars_multirun_with_no_scalars(self):
+        server = self.load_server([self._RUN_WITH_HISTOGRAM])
+        response = server.post(
+            "/data/plugin/scalars/scalars_multirun",
+            data={
+                "query": json.dumps(
+                    {
+                        "tag": "%s/scalar_summary" % self._SCALAR_TAG,
+                        "runs": [self._RUN_WITH_HISTOGRAM],
+                    }
+                )
+            },
+        )
+        self.assertEqual(200, response.status_code)
+        self.assertEqual("application/json", response.headers["Content-Type"])
+        r = json.loads(response.get_data())
+        self.assertEmpty(r)
+
+    def test_scalars_multirun_with_empty_run(self):
+        server = self.load_server([self._RUN_WITH_NOTHING])
+        response = server.post(
+            "/data/plugin/scalars/scalars_multirun",
+            data={
+                "query": json.dumps(
+                    {
+                        "tag": "%s/scalar_summary" % self._SCALAR_TAG,
+                        "runs": [self._RUN_WITH_NOTHING],
+                    }
+                )
+            },
+        )
+        self.assertEqual(200, response.status_code)
+        self.assertEqual("application/json", response.headers["Content-Type"])
+        r = json.loads(response.get_data())
+        self.assertEmpty(r)
+
     def test_scalars_with_histogram(self):
         server = self.load_server([self._RUN_WITH_HISTOGRAM])
         response = server.get(

tensorboard/plugins/scalar/tf_scalar_dashboard/tf-scalar-card.html

@@ -45,8 +45,8 @@
       <tf-line-chart-data-loader
         active="[[active]]"
         color-scale="[[_getColorScale(colorScale)]]"
-        data-series="[[_getDataSeries(dataToLoad.*)]]"
-        data-to-load="[[dataToLoad]]"
+        data-series="[[_getDataSeries(tagAndRuns)]]"
+        data-to-load="[[_asSingletonArray(tagAndRuns)]]"
         get-data-load-name="[[_getDataLoadName]]"
         get-data-load-url="[[getDataLoadUrl]]"
         request-data="[[requestData]]"
@@ -102,7 +102,7 @@
       <template is="dom-if" if="[[showDownloadLinks]]">
         <div class="download-links">
           <tf-downloader
-            runs="[[_runsFromData(dataToLoad)]]"
+            runs="[[tagAndRuns.runs]]"
             tag="[[tag]]"
             url-fn="[[_downloadUrlFn]]"
           ></tf-downloader>
@@ -188,10 +188,8 @@
       properties: {
         tag: String,
 
-        /**
-         * @type {Array<Object>}
-         */
-        dataToLoad: Array,
+        // {tag: string, runs: string[]}
+        tagAndRuns: Object,
 
         /**
          * @type {vz_chart_helpers.XType}
@@ -218,14 +216,19 @@
           type: Object,
           value: function() {
             return (scalarChart, datum, data) => {
-              const formattedData = data.map((datum) => ({
-                wall_time: new Date(datum[0] * 1000),
-                step: datum[1],
-                scalar: datum[2],
-              }));
-              const name = this._getSeriesNameFromDatum(datum);
-              scalarChart.setSeriesMetadata(name, datum);
-              scalarChart.setSeriesData(name, formattedData);
+              for (const [run, points] of Object.entries(data)) {
+                const formattedData = points.map((point) => ({
+                  wall_time: new Date(point[0] * 1000),
+                  step: point[1],
+                  scalar: point[2],
+                }));
+                const name = this._getSeriesNameForRun(run);
+                scalarChart.setSeriesMetadata(name, {
+                  run,
+                  tag: datum.tag,
+                });
+                scalarChart.setSeriesData(name, formattedData);
+              }
               scalarChart.commitChanges();
             };
           },
@@ -235,14 +238,12 @@
         getDataLoadUrl: {
           type: Function,
           value: function() {
-            return ({tag, run}) => {
-              return tf_backend
+            return ({tag, runs}) => {
+              const url = tf_backend
                 .getRouter()
-                .pluginRoute(
-                  'scalars',
-                  '/scalars',
-                  new URLSearchParams({tag, run})
-                );
+                .pluginRoute('scalars', '/scalars_multirun');
+              const postData = {query: JSON.stringify({tag, runs})};
+              return {url, postData};
             };
           },
         },
@@ -266,7 +267,7 @@
         _getDataLoadName: {
           type: Function,
           value: function() {
-            return (datum) => this._getSeriesNameFromDatum(datum);
+            return (datum) => datum.tag;
           },
         },
 
@@ -286,8 +287,7 @@
             columns.splice(ind, 1, {
               title: 'Name',
               evaluate: (d) => {
-                const datum = d.dataset.metadata().meta;
-                return this._getSeriesDisplayNameFromDatum(datum);
+                return d.dataset.metadata().meta.run;
               },
             });
             return columns;
@@ -321,19 +321,12 @@
         // specifier, truncating the SVG. (See issue #1874.)
         this.$$('#svgLink').href = `data:image/svg+xml;base64,${btoa(svgStr)}`;
       },
-      _runsFromData(data) {
-        return data.map((datum) => datum.run);
-      },
-      _getDataSeries() {
-        return this.dataToLoad.map((d) => this._getSeriesNameFromDatum(d));
+      _getDataSeries(tagAndRuns) {
+        return tagAndRuns.runs.map((run) => this._getSeriesNameForRun(run));
       },
       // name is a stable identifier for a series.
-      _getSeriesNameFromDatum({run, experiment = {name: '_default'}}) {
-        return JSON.stringify([experiment.name, run]);
-      },
-      // title is a visible string of a series for the UI.
-      _getSeriesDisplayNameFromDatum(datum) {
-        return datum.run;
+      _getSeriesNameForRun(run) {
+        return JSON.stringify(run);
       },
       _getColorScale() {
         if (this.colorScale !== null) {
@@ -343,11 +336,14 @@
         // defined in tf_color_scale.
         return {
           scale: (name) => {
-            const [exp, run] = JSON.parse(name);
+            const run = JSON.parse(name);
             return tf_color_scale.runsColorScale(run);
           },
         };
       },
+      _asSingletonArray(x) {
+        return [x];
+      },
     });
   </script>
 </dom-module>

tensorboard/plugins/scalar/tf_scalar_dashboard/tf-scalar-dashboard.html

@@ -162,7 +162,7 @@ <h3>No scalar data was found.</h3>
               <template>
                 <tf-scalar-card
                   active="[[active]]"
-                  data-to-load="[[item.series]]"
+                  tag-and-runs="[[item]]"
                   ignore-y-outliers="[[_ignoreYOutliers]]"
                   multi-experiments="[[_getMultiExperiments(dataSelection)]]"
                   request-manager="[[_requestManager]]"
@@ -361,7 +361,7 @@ <h3>No scalar data was found.</h3>
         categories.forEach((category) => {
           category.items = category.items.map((item) => ({
             tag: item.tag,
-            series: item.runs.map((run) => ({run, tag: item.tag})),
+            runs: item.runs,
           }));
         });
         this.updateArrayProp('_categories', categories, this._getCategoryKey);
@@ -370,7 +370,7 @@ <h3>No scalar data was found.</h3>
       _tagMetadata(category, runToTagsInfo, item) {
         const tag = item.tag;
         const runToTagInfo = {};
-        item.series.forEach(({run}) => {
+        item.runs.forEach((run) => {
           runToTagInfo[run] = runToTagsInfo[run][tag];
         });
         // All new-style scalar tags include the `/scalar_summary`