Model comparison v2

tensorboard/plugins/interactive_inference/interactive_inference_plugin.py

@@ -436,24 +436,24 @@ def _infer_mutants_handler(self, request):
       (inference_addresses, model_names, model_versions,
           model_signatures) = self._parse_request_arguments(request)
 
-      # TODO(tolgab) Generalize this to multiple models
-      model_num = 0
-      serving_bundle = inference_utils.ServingBundle(
-          inference_addresses[model_num],
-          model_names[model_num],
-          request.args.get('model_type'),
-          model_versions[model_num],
-          model_signatures[model_num],
-          request.args.get('use_predict') == 'true',
-          request.args.get('predict_input_tensor'),
-          request.args.get('predict_output_tensor'))
+      serving_bundles = []
+      for model_num in xrange(len(inference_addresses)):
+        serving_bundles.append(inference_utils.ServingBundle(
+            inference_addresses[model_num],
+            model_names[model_num],
+            request.args.get('model_type'),
+            model_versions[model_num],
+            model_signatures[model_num],
+            request.args.get('use_predict') == 'true',
+            request.args.get('predict_input_tensor'),
+            request.args.get('predict_output_tensor')))
 
       viz_params = inference_utils.VizParams(
           request.args.get('x_min'), request.args.get('x_max'),
           self.examples[0:NUM_EXAMPLES_TO_SCAN], NUM_MUTANTS,
           request.args.get('feature_index_pattern'))
       json_mapping = inference_utils.mutant_charts_for_feature(
-          examples, feature_name, serving_bundle, viz_params)
+          examples, feature_name, serving_bundles, viz_params)
       return http_util.Respond(request, json_mapping, 'application/json')
     except common_utils.InvalidUserInputError as e:
       return http_util.Respond(request, {'error': e.message},

tensorboard/plugins/interactive_inference/interactive_inference_plugin_test.py

@@ -194,15 +194,15 @@ def test_infer_mutants_handler(self, mock_mutant_charts_for_feature):
     # A no-op that just passes the example passed to mutant_charts_for_feature
     # back through. This tests that the URL parameters get processed properly
     # within infer_mutants_handler.
-    def pass_through(example, feature_name, serving_bundle, viz_params):
+    def pass_through(example, feature_name, serving_bundles, viz_params):
       return {
           'example': str(example),
           'feature_name': feature_name,
-          'serving_bundle': {
-              'inference_address': serving_bundle.inference_address,
-              'model_name': serving_bundle.model_name,
-              'model_type': serving_bundle.model_type,
-          },
+          'serving_bundles': [{
+              'inference_address': serving_bundles[0].inference_address,
+              'model_name': serving_bundles[0].model_name,
+              'model_type': serving_bundles[0].model_type,
+          }],
           'viz_params': {
               'x_min': viz_params.x_min,
               'x_max': viz_params.x_max
@@ -229,10 +229,10 @@ def pass_through(example, feature_name, serving_bundle, viz_params):
     self.assertEqual(str([example]), result['example'])
     self.assertEqual('single_int', result['feature_name'])
     self.assertEqual('ml-serving-temp.prediction',
-                     result['serving_bundle']['inference_address'])
+                     result['serving_bundles'][0]['inference_address'])
     self.assertEqual('/ml/cassandrax/iris_classification',
-                     result['serving_bundle']['model_name'])
-    self.assertEqual('classification', result['serving_bundle']['model_type'])
+                     result['serving_bundles'][0]['model_name'])
+    self.assertEqual('classification', result['serving_bundles'][0]['model_type'])
     self.assertAlmostEqual(-10, result['viz_params']['x_min'])
     self.assertAlmostEqual(10, result['viz_params']['x_max'])
 

tensorboard/plugins/interactive_inference/tf_interactive_inference_dashboard/demo/tf-interactive-inference-age-demo.html

@@ -45,6 +45,7 @@
       },
       ready: async function() {
         this.$.dash.settingsClicked_();
+        this.$.dash.updateNumberOfModels_();
         this.means = {
           'age': 38.64358543876172,
           'education-num': 10.078088530363212,
@@ -267,7 +268,7 @@
                 results.push({step: step, scalar: adjustedScore});
               }
               this.$.dash.makeChartForFeature_(isNum ? 'numeric' : 'categorical',
-                  e.detail.feature_name, [{'value': results}]);
+                  e.detail.feature_name, [[{'value': results}]]);
             };
             setTimeout(method, 50);
           });

tensorboard/plugins/interactive_inference/tf_interactive_inference_dashboard/demo/tf-interactive-inference-demo.html

@@ -46,6 +46,7 @@
       },
       ready: async function() {
         this.$.dash.settingsClicked_();
+        this.$.dash.updateNumberOfModels_();
         this.means = {
           'age': 38.64358543876172,
           'education-num': 10.078088530363212,
@@ -276,7 +277,7 @@
 
               this.$.dash.makeChartForFeature_(
                 isNum ? 'numeric' : 'categorical',
-                e.detail.feature_name, [{'1': results}]);
+                e.detail.feature_name, [[{'1': results}]]);
             };
             setTimeout(method, 50);
           });

tensorboard/plugins/interactive_inference/tf_interactive_inference_dashboard/demo/tf-interactive-inference-image-demo.html

@@ -53,7 +53,7 @@
       },
       ready: async function() {
         this.$.dash.settingsClicked_();
-
+        this.$.dash.updateNumberOfModels_();
         this.model = await tf.loadModel(tf.io.browserHTTPRequest(
           'data/images/model.json', {credentials: 'include'}));
         const DATA_PATH = "data/images/smile_examples.json";

tensorboard/plugins/interactive_inference/tf_interactive_inference_dashboard/demo/tf-interactive-inference-iris-demo.html

@@ -47,6 +47,7 @@
       ready: async function() {
         this.$.dash.multiClass = true;
         this.$.dash.settingsClicked_();
+        this.$.dash.updateNumberOfModels_();
         this.model = await tf.loadModel(tf.io.browserHTTPRequest(
           'data/iris/model.json', {credentials: 'include'}));
         const DATA_PATH = "data/iris/iris.json";
@@ -209,7 +210,7 @@
               this.$.dash.makeChartForFeature_(
                 isNum ? 'numeric' : 'categorical',
                 e.detail.feature_name,
-                [{'0': results[0]}, {'1': results[1]},{'2': results[2]}]);
+                [[{'0': results[0], '1': results[1], '2': results[2]}]]);
             };
             setTimeout(method, 50);
           })

tensorboard/plugins/interactive_inference/tf_interactive_inference_dashboard/demo/tf-interactive-inference-multi-demo.html

@@ -30,7 +30,7 @@
         width: 100%;
       }
     </style>
-    <tf-interactive-inference-dashboard id="dash" local="true" label-vocab="[[labelVocab]]"  model-name="demo" inference-address="demo">
+    <tf-interactive-inference-dashboard id="dash" local="true" label-vocab="[[labelVocab]]"  model-name="demo1, demo2" inference-address="demo1, demo2">
     </tf-interactive-inference-dashboard>
   </template>
   <script>
@@ -46,6 +46,7 @@
       },
       ready: async function() {
         this.$.dash.settingsClicked_();
+        this.$.dash.updateNumberOfModels_();
         this.means = {
           'age': 38.64358543876172,
           'education-num': 10.078088530363212,
@@ -239,7 +240,9 @@
               const PRED_SIZE = 2;
               const BATCH_SIZE = 128
               const results = [];
+              const results2 = [];
               const predValuesList = [];
+              const predValuesList2 = [];
               for (let i = 0; i < examples.length; i += BATCH_SIZE) {
                 let tlist = [];
                 for (let idx = i;
@@ -251,39 +254,51 @@
                 tlist.forEach(tensor => tensor.dispose());
                 const input = tconcat.reshape([tlist.length, 104]);
                 const res = this.model.predict(input, {batchSize: BATCH_SIZE});
+                const res2 = this.model2.predict(input, {batchSize: BATCH_SIZE});
                 const vals = await res.data();
+                const vals2 = await res2.data();
                 predValuesList.push(vals);
+                predValuesList2.push(vals2);
                 input.dispose();
                 res.dispose();
+                res2.dispose();
                 tconcat.dispose();
               }
               const predSize = predValuesList.reduce((a, b) => a + b.length, 0);
               const predValues = new Float32Array(predSize);
+              const predValues2 = new Float32Array(predSize);
               let curIdx = 0;
               for (let i = 0; i < predValuesList.length; i++) {
                 predValues.set(predValuesList[i], curIdx);
+                predValues2.set(predValuesList2[i], curIdx);
                 curIdx += predValuesList[i].length;
               }
               for (let trialIdx = 0; trialIdx < numTrials; trialIdx++) {
                 const startingPredIdx =
                   trialIdx * exampleIndices.length * PRED_SIZE;
                 const scores = [];
+                const scores2 = [];
                 for (let exampleIdx = 0; exampleIdx < exampleIndices.length;
                     exampleIdx++) {
                   scores.push(
                     predValues[startingPredIdx + (exampleIdx * PRED_SIZE)]);
+                  scores2.push(
+                    predValues2[startingPredIdx + (exampleIdx * PRED_SIZE)]);
                 }
                 const score = scores.reduce(
                   (prev, cur) => prev + cur, 0) / scores.length;
+                const score2 = scores2.reduce(
+                  (prev, cur) => prev + cur, 0) / scores.length;
                 const ex = examples[trialIdx * exampleIndices.length];
                 const step = isNum ? ex.features.feature[
                   e.detail.feature_name].floatList.value[0]
                       : [atob(ex.features.feature[
                           e.detail.feature_name].bytesList.value[0])];
                 results.push({step: step, scalar: score});
+                results2.push({step: step, scalar: score2});
               }
               this.$.dash.makeChartForFeature_(isNum ? 'numeric' : 'categorical',
-                  e.detail.feature_name, [{'1': results}]);
+                  e.detail.feature_name, [[{'1': results}, {'1': results2}]]);
             };
             setTimeout(method, 50);
           });