projector: add images to nearest neighbors list (#2543)

Summary:
In addition to showing the labels of nearest neighbors, the projector
also shows sprites associated with those points. Sometimes the label
data alone is only minimally useful (e.g., when all the labels are the
same), and the sprites provide much greater insight. The additional
detail due to the larger view is also particularly helpful on datasets
with higher-resolution sprites, like audio spectrograms.

The behavior is enabled by default, and can be disabled via a new “show
images” checkbox.

Addresses part of #2492. 

Screenshots:

  - <https://user-images.githubusercontent.com/4317806/63119171-570ae280-bf54-11e9-8e83-80e19701a60f.png>
  - <https://user-images.githubusercontent.com/6741720/62971619-4a657d80-bde0-11e9-84d2-f7b9f56b8bf6.png>

Test Plan:
Launch the projector on a dataset without images, and observe that the
behavior is the same as before: no images, no new checkbox, no error
messages. Then, switch to a dataset with images, and select a point.
Verify that the “show images” toggle works, that the images are properly
rendered, and that the images update when you select a new data point.

Author: beasteers

tensorboard/plugins/projector/vz_projector/vz-projector-inspector-panel.html

@@ -124,6 +124,16 @@
         border-left: 1px solid rgba(0, 0, 0, 0.15);
       }
 
+      .nn-list .sprite-image,
+      .metadata-list .sprite-image {
+        width: 100%;
+      }
+
+      .nn-list.nn-img-show .sprite-image,
+      .metadata-list.nn-img-show .sprite-image {
+        display: block;
+      }
+
       .nn-list .neighbor-link:hover,
       .metadata-list .metadata-link:hover {
         cursor: pointer;
@@ -146,6 +156,11 @@
         float: right;
       }
 
+      .neighbor-image-controls {
+        display: flex;
+        padding: 0.8em 0.1em;
+      }
+
       .options a {
         color: #727272;
         font-size: 13px;
@@ -158,7 +173,7 @@
       }
 
       .neighbors {
-        margin-bottom: 30px;
+        margin-bottom: 15px;
       }
 
       .neighbors-options {
@@ -206,6 +221,12 @@
         display: flex;
         flex-direction: column;
       }
+
+      .results,
+      .nn,
+      .nn-list {
+        flex: 1 0 100px;
+      }
     </style>
     <div class="container">
       <div class="buttons">
@@ -267,6 +288,24 @@
               <a class="euclidean" href="javascript:void(0);">EUCLIDEAN</a>
             </div>
           </div>
+          <div class="neighbor-image-controls">
+            <template is="dom-if" if="[[spriteImagesAvailable]]">
+              <paper-checkbox checked="{{showNeighborImages}}">
+                show images
+                <paper-icon-button
+                  icon="help"
+                  class="help-icon"
+                ></paper-icon-button>
+                <paper-tooltip
+                  position="bottom"
+                  animation-delay="0"
+                  fit-to-visible-bounds
+                >
+                  Show the images of the nearest neighbors.
+                </paper-tooltip>
+              </paper-checkbox>
+            </template>
+          </div>
         </div>
         <p>Nearest points in the original space:</p>
         <div class="nn-list"></div>

tensorboard/plugins/projector/vz_projector/vz-projector-inspector-panel.ts

@@ -15,6 +15,7 @@ limitations under the License.
 namespace vz_projector {
   /** Limit the number of search results we show to the user. */
   const LIMIT_RESULTS = 100;
+  const DEFAULT_NEIGHBORS = 100;
 
   // tslint:disable-next-line
   export let InspectorPanelPolymer = PolymerElement({
@@ -23,11 +24,29 @@ namespace vz_projector {
       selectedMetadataField: String,
       metadataFields: Array,
       metadataColumn: String,
-      numNN: {type: Number, value: 100},
+      numNN: {type: Number, value: DEFAULT_NEIGHBORS},
       updateNumNN: Object,
+      spriteMeta: Object, // type: `SpriteMetadata`
+      showNeighborImages: {
+        type: Boolean,
+        value: true,
+        observer: '_refreshNeighborsList',
+      },
+      spriteImagesAvailable: {
+        type: Boolean,
+        value: true,
+        observer: '_refreshNeighborsList',
+      },
     },
   });
 
+  type SpriteMetadata = {
+    imagePath?: string;
+    singleImageDim?: number[];
+    aspectRatio?: number;
+    nCols?: number;
+  };
+
   export class InspectorPanel extends InspectorPanelPolymer {
     distFunc: DistanceFunction;
     numNN: number;
@@ -37,10 +56,13 @@ namespace vz_projector {
     private selectedMetadataField: string;
     private metadataFields: string[];
     private metadataColumn: string;
+    private spriteMeta: SpriteMetadata;
     private displayContexts: string[];
     private projector: Projector;
     private selectedPointIndices: number[];
     private neighborsOfFirstPoint: knn.NearestEntry[];
+    private showNeighborImages: boolean;
+    private spriteImagesAvailable: boolean;
     private searchBox: ProjectorInput;
 
     private resetFilterButton: HTMLButtonElement;
@@ -106,6 +128,26 @@ namespace vz_projector {
         return stats.name;
       });
 
+      if (
+        spriteAndMetadata.spriteMetadata &&
+        spriteAndMetadata.spriteMetadata.imagePath
+      ) {
+        const [
+          spriteWidth,
+          spriteHeight,
+        ] = spriteAndMetadata.spriteMetadata.singleImageDim;
+
+        this.spriteMeta = {
+          imagePath: spriteAndMetadata.spriteImage.src,
+          aspectRatio: spriteWidth / spriteHeight,
+          nCols: Math.floor(spriteAndMetadata.spriteImage.width / spriteWidth),
+          singleImageDim: [spriteWidth, spriteHeight],
+        };
+      } else {
+        this.spriteMeta = {};
+      }
+      this.spriteImagesAvailable = !!this.spriteMeta.imagePath;
+
       if (
         this.selectedMetadataField == null ||
         this.metadataFields.filter(
@@ -127,6 +169,10 @@ namespace vz_projector {
       this.enableResetFilterButton(false);
     }
 
+    _refreshNeighborsList() {
+      this.updateNeighborsList();
+    }
+
     metadataEditorContext(enabled: boolean, metadataColumn: string) {
       if (!this.projector || !this.projector.dataSet) {
         return;
@@ -269,7 +315,40 @@ namespace vz_projector {
       return point.metadata[this.selectedMetadataField].toString();
     }
 
-    private updateNeighborsList(neighbors: knn.NearestEntry[]) {
+    private spriteImageRenderer() {
+      const spriteImagePath = this.spriteMeta.imagePath;
+      const {aspectRatio, nCols} = this.spriteMeta;
+      const paddingBottom = 100 / aspectRatio + '%';
+      const backgroundSize = `${nCols * 100}% ${nCols * 100}%`;
+      const backgroundImage = `url(${CSS.escape(spriteImagePath)})`;
+
+      return (neighbor: knn.NearestEntry): HTMLElement => {
+        const spriteElementImage = document.createElement('div');
+        spriteElementImage.className = 'sprite-image';
+        spriteElementImage.style.backgroundImage = backgroundImage;
+        spriteElementImage.style.paddingBottom = paddingBottom;
+        spriteElementImage.style.backgroundSize = backgroundSize;
+        const [row, col] = [
+          Math.floor(neighbor.index / nCols),
+          neighbor.index % nCols,
+        ];
+        const [top, left] = [
+          (row / (nCols - 1)) * 100,
+          (col / (nCols - 1)) * 100,
+        ];
+        spriteElementImage.style.backgroundPosition = `${left}% ${top}%`;
+
+        return spriteElementImage;
+      };
+    }
+
+    private updateNeighborsList(neighbors?: knn.NearestEntry[]) {
+      neighbors = neighbors || this._currentNeighbors;
+      this._currentNeighbors = neighbors;
+      if (neighbors == null) {
+        return;
+      }
+
       const nnlist = this.$$('.nn-list') as HTMLDivElement;
       nnlist.innerHTML = '';
 
@@ -282,6 +361,10 @@ namespace vz_projector {
       this.searchBox.message = '';
       const minDist = neighbors.length > 0 ? neighbors[0].dist : 0;
 
+      if (this.spriteImagesAvailable && this.showNeighborImages) {
+        var imageRenderer = this.spriteImageRenderer();
+      }
+
       for (let i = 0; i < neighbors.length; i++) {
         const neighbor = neighbors[i];
 
@@ -332,6 +415,11 @@ namespace vz_projector {
           barElement.appendChild(tickElement);
         }
 
+        if (this.spriteImagesAvailable && this.showNeighborImages) {
+          const neighborElementImage = imageRenderer(neighbor);
+          neighborElement.appendChild(neighborElementImage);
+        }
+
         neighborElementLink.appendChild(labelValueElement);
         neighborElementLink.appendChild(barElement);
         neighborElement.appendChild(neighborElementLink);