Skip to content

Commit c9c5c6b

Browse files
authored
Fix saliency viewing in example viewer (#1472)
* Add flags to TBContext * Add flags to TBContext in application * Add flags to tbcontext * gitignore changes * jwexler updated rules_closure version * Fix saliency viewing in example viewer * review updates
1 parent f0439f9 commit c9c5c6b

File tree

2 files changed

+77
-34
lines changed

2 files changed

+77
-34
lines changed

tensorboard/components/vz_example_viewer/vz-example-viewer.html

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
display: block;
7777
padding: 8px;
7878
width: fit-content;
79+
border: 1px solid lightgrey;
7980
}
8081

8182
.hide-saliency-controls {
@@ -474,7 +475,7 @@
474475
value="[[getFirstFeatureValue(feat.name)]]">
475476
</input>
476477
<template is="dom-if" if="[[featureHasMultipleValues(feat.name)]]">
477-
<paper-button data-feature="[[feat.name]]" on-click="expandFeature" class$="[[getInputPillClass(expandPillClass, displayMode)]]">
478+
<paper-button data-feature="[[feat.name]]" on-click="expandFeature" class$="[[getInputPillClass(feat.name, displayMode)]]">
478479
...
479480
</paper-button>
480481
</template>
@@ -498,7 +499,7 @@
498499
value="[[getFirstCompareFeatureValue(feat.name)]]">
499500
</input>
500501
<template is="dom-if" if="[[compareFeatureHasMultipleValues(feat.name)]]">
501-
<paper-button data-feature="[[feat.name]]" on-click="expandFeature" class$="[[getCompareInputClass(expandPillClass, displayMode)]]">
502+
<paper-button data-feature="[[feat.name]]" on-click="expandFeature" class$="[[getCompareInputClass(feat.name, displayMode)]]">
502503
...
503504
</paper-button>
504505
</template>
@@ -635,7 +636,7 @@
635636
value="[[getFirstSeqFeatureValue(seqfeat.name, seqNumber)]]">
636637
</input>
637638
<template is="dom-if" if="[[seqFeatureHasMultipleValues(seqfeat.name, seqNumber)]]">
638-
<paper-button data-feature="[[seqfeat.name]]" on-click="expandFeature" class$="[[getInputPillClass(expandPillClass, displayMode)]]">
639+
<paper-button data-feature="[[seqfeat.name]]" on-click="expandFeature" class$="[[getInputPillClass(seqfeat.name, displayMode)]]">
639640
...
640641
</paper-button>
641642
</template>
@@ -661,7 +662,7 @@
661662
value="[[getFirstCompareSeqFeatureValue(seqfeat.name, seqNumber)]]">
662663
</input>
663664
<template is="dom-if" if="[[compareSeqFeatureHasMultipleValues(seqfeat.name, seqNumber)]]">
664-
<paper-button data-feature="[[seqfeat.name]]" on-click="expandFeature" class$="[[getSeqCompareInputClass(expandPillClass, displayMode, seqNumber)]]">
665+
<paper-button data-feature="[[seqfeat.name]]" on-click="expandFeature" class$="[[getSeqCompareInputClass(seqfeat.name, displayMode, seqNumber)]]">
665666
...
666667
</paper-button>
667668
</template>
@@ -685,8 +686,8 @@
685686
</template>
686687
<paper-icon-button on-click="openAddFeatureDialog" icon="add" title="Add feature"
687688
class$="[[getAddValueButtonClass(readonly)]]">
688-
</paper-icon button>
689-
<paper-card class$="[[getSaliencyControlsHolderClass(saliency)]]">
689+
</paper-icon-button>
690+
<div class$="[[getSaliencyControlsHolderClass(saliency)]]">
690691
<div class="saliency-legend-label">Saliency legend</div>
691692
<div class="flex-controls">
692693
<svg id="saliencyLegend"></svg>
@@ -700,7 +701,7 @@
700701
value="[[saliencyCutoff]]">
701702
</paper-slider>
702703
</div>
703-
</paper-card>
704+
</div>
704705
</div>
705706
<paper-icon-button id="deletevalue" icon="delete" class$="[[getDeleteValueButtonClass(readonly, showDeleteValueButton)]]"
706707
data-feature="[[focusedFeatureName]]" data-index="[[focusedFeatureValueIndex]]"

tensorboard/components/vz_example_viewer/vz-example-viewer.ts

Lines changed: 69 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ const clipSaliencyRatio = .95;
8787
// Colors for the saliency color scale.
8888
const posSaliencyColor = '#0f0';
8989
const negSaliencyColor = '#f00';
90-
const neutralSaliencyColor = '#d3d3d3';
90+
const neutralSaliencyColor = '#e8eaed';
9191

9292

9393
const COLOR_INTERPOLATOR = d3.interpolateRgb;
@@ -129,7 +129,7 @@ Polymer({
129129
ignoreChange: Boolean,
130130
minSal: {type: Number, value: 0},
131131
maxSal: {type: Number, value: 0},
132-
showSaliency: {type: Boolean, value: false},
132+
showSaliency: {type: Boolean, value: true},
133133
imageInfo: {type: Object, value: {}},
134134
windowWidth: {type: Number, value: DEFAULT_WINDOW_WIDTH},
135135
windowCenter: {type: Number, value: DEFAULT_WINDOW_CENTER},
@@ -145,14 +145,13 @@ Polymer({
145145
colors: {type: Object, computed: 'getColors(saliency)', observer: 'createLegend'},
146146
displayMode: {type: String, value: 'grid'},
147147
featureSearchValue: {type: String, value: '', notify: true},
148-
filteredFeaturesList: {type: Object, computed: 'getFilteredFeaturesList(featuresList, featureSearchValue)'},
149-
filteredSeqFeaturesList: {type: Object, computed: 'getFilteredFeaturesList(seqFeaturesList, featureSearchValue)'},
148+
filteredFeaturesList: {type: Object, computed: 'getFilteredFeaturesList(featuresList, featureSearchValue, saliency)'},
149+
filteredSeqFeaturesList: {type: Object, computed: 'getFilteredFeaturesList(seqFeaturesList, featureSearchValue, saliency)'},
150150
focusedFeatureName: String,
151151
focusedFeatureValueIndex: Number,
152152
focusedSeqNumber: Number,
153153
showDeleteValueButton: {type: Boolean, value: false},
154154
expandedFeatures: {type: Object, value: {}},
155-
expandPillClass: {type: String, value: 'expandPill'},
156155
expandAllFeatures: {type: Boolean, value: false},
157156
zeroIndex: {type: Number, value: 0},
158157
compareJson: {type: Object, observer: 'createCompareExamplesFromJson'},
@@ -172,7 +171,7 @@ Polymer({
172171
compareTitle: String,
173172
},
174173
observers: [
175-
'haveSaliency(featuresList, saliency, colors, showSaliency, saliencyCutoff)',
174+
'haveSaliency(filteredFeaturesList, saliency, colors, showSaliency, saliencyCutoff)',
176175
'seqSaliency(seqNumber, seqFeaturesList, saliency, colors, showSaliency, saliencyCutoff)',
177176
],
178177

@@ -276,8 +275,20 @@ Polymer({
276275
},
277276

278277
getFilteredFeaturesList: function(featureList: NameAndFeature[],
279-
searchValue: string) {
278+
searchValue: string, saliency: SaliencyMap) {
280279
let filtered = featureList;
280+
const checkSal = saliency && Object.keys(saliency).length > 0;
281+
// Create a dict of feature names to the total absolute saliency of all
282+
// its feature values, to sort features with the most salienct features at
283+
// the top.
284+
const saliencyTotals = checkSal ?
285+
Object.assign({}, ...Object.keys(saliency).map(
286+
name => ({[name]: typeof saliency[name] == 'number' ?
287+
Math.abs(saliency[name] as number) :
288+
(saliency[name] as Array<number>).reduce((total, cur) =>
289+
Math.abs(total) + Math.abs(cur) , 0)}))) :
290+
{};
291+
281292
if (searchValue != '') {
282293
const re = new RegExp(searchValue, 'i');
283294
filtered = featureList.filter(feature => re.test(feature.name));
@@ -288,6 +299,18 @@ Polymer({
288299
} else if (this.isImage(b.name) && !this.isImage(a.name)) {
289300
return 1;
290301
} else {
302+
if (checkSal) {
303+
if (a.name in saliency && !(b.name in saliency)) {
304+
return -1;
305+
} else if (b.name in saliency && !(a.name in saliency)) {
306+
return 1;
307+
} else {
308+
const diff = saliencyTotals[b.name] - saliencyTotals[a.name];
309+
if (diff != 0) {
310+
return diff;
311+
}
312+
}
313+
}
291314
return a.name.localeCompare(b.name);
292315
}
293316
});
@@ -326,29 +349,32 @@ Polymer({
326349
]);
327350
},
328351

352+
selectAll: function(query: string) {
353+
return d3.selectAll(
354+
Polymer.dom(this.root).querySelectorAll(query) as any);
355+
},
356+
329357
haveSaliency: function() {
330-
if (!this.featuresList || !this.saliency ||
358+
if (!this.filteredFeaturesList || !this.saliency ||
331359
Object.keys(this.saliency).length === 0 || !this.colors) {
332360
return;
333361
}
334362

335363
// TODO(jwexler): Find a way to do this without requestAnimationFrame.
336-
// If the paper-inputs for the features have yet to be rendered, wait to
337-
// perform this processing. There should be paper-inputs for all non-image
364+
// If the inputs for the features have yet to be rendered, wait to
365+
// perform this processing. There should be inputs for all non-image
338366
// features.
339-
if (d3.selectAll('.value input').size() <
340-
(this.featuresList.length - Object.keys(this.imageInfo).length)) {
367+
if (this.selectAll('input.value-pill').size() <
368+
(this.filteredFeaturesList.length - Object.keys(this.imageInfo).length)) {
341369
requestAnimationFrame(() => this.haveSaliency());
342370
return;
343371
}
344372

345-
// Reset all text to black
346-
d3.selectAll<HTMLInputElement, {}>('.value-pill')
347-
.style('background', 'lightgrey');
348-
373+
// Reset all backgrounds to the neutral color.
374+
this.selectAll('.value-pill').style('background', neutralSaliencyColor);
349375
// Color the text of each input element of each feature according to the
350376
// provided saliency information.
351-
for (const feat of this.featuresList) {
377+
for (const feat of this.filteredFeaturesList) {
352378
const val = this.saliency[feat.name] as SaliencyValue;
353379
// If there is no saliency information for the feature, do not color it.
354380
if (!val) {
@@ -357,13 +383,28 @@ Polymer({
357383
const colorFn = Array.isArray(val) ?
358384
(d: {}, i: number) => this.getColorForSaliency(val[i]) :
359385
() => this.getColorForSaliency(val);
360-
361-
d3.selectAll<HTMLInputElement, {}>(
362-
`.${this.sanitizeFeature(feat.name)}.value-pill`)
363-
.style('background', this.showSaliency ? colorFn : () => 'lightgrey');
386+
this.selectAll(
387+
`input.${this.sanitizeFeature(feat.name)}.value-pill`)
388+
.style('background',
389+
this.showSaliency ? colorFn : () => neutralSaliencyColor);
390+
391+
// Color the "more feature values" button with the most extreme saliency
392+
// of any of the feature values hidden behind the button.
393+
if (Array.isArray(val)) {
394+
const valArray = val as Array<number>;
395+
const moreButton = this.selectAll(
396+
`paper-button.${this.sanitizeFeature(feat.name)}.value-pill`);
397+
let mostExtremeSal = 0;
398+
for (let i = 1; i < valArray.length; i++) {
399+
if (Math.abs(valArray[i]) > Math.abs(mostExtremeSal)) {
400+
mostExtremeSal = valArray[i];
401+
}
402+
}
403+
moreButton.style('background', this.showSaliency ?
404+
() => this.getColorForSaliency(mostExtremeSal) :
405+
() => neutralSaliencyColor);
406+
}
364407
}
365-
// TODO(jwexler): Determine how to set non-fixed widths to input boxes
366-
// inside of grid iron-list.
367408
},
368409

369410
/**
@@ -382,7 +423,7 @@ Polymer({
382423
// TODO(jwexler): Find a way to do this without requestAnimationFrame.
383424
// If the paper-inputs for the features have yet to be rendered, wait to
384425
// perform this processing.
385-
if (d3.selectAll('.value input').size() < this.seqFeaturesList.length) {
426+
if (this.selectAll('.value input').size() < this.seqFeaturesList.length) {
386427
requestAnimationFrame(() => this.seqSaliency());
387428
return;
388429
}
@@ -401,7 +442,7 @@ Polymer({
401442
(d: {}, i: number) => this.getColorForSaliency(val[i]) :
402443
() => this.getColorForSaliency(val);
403444

404-
d3.selectAll<HTMLInputElement, {}>(
445+
this.selectAll(
405446
`.${this.sanitizeFeature(feat.name)} input`)
406447
.style('color', this.showSaliency ? colorFn : () => 'black');
407448
}
@@ -954,6 +995,7 @@ Polymer({
954995
this.ignoreChange = false;
955996
setTimeout(() => {
956997
this.example = temp;
998+
this.haveSaliency();
957999
}, 0);
9581000
},
9591001

@@ -1057,7 +1099,7 @@ Polymer({
10571099
const legendSvg = d3.select(this.$.saliencyLegend).append('g');
10581100
const gradient = legendSvg.append('defs')
10591101
.append('linearGradient')
1060-
.attr('id', 'gradient')
1102+
.attr('id', 'vzexampleviewergradient')
10611103
.attr("x1", "0%")
10621104
.attr("y1", "0%")
10631105
.attr("x2", "100%")
@@ -1106,7 +1148,7 @@ Polymer({
11061148
.attr('y1', 0)
11071149
.attr('width', LEGEND_WIDTH_PX)
11081150
.attr('height', LEGEND_HEIGHT_PX)
1109-
.style('fill', 'url(#gradient)');
1151+
.style('fill', 'url(#vzexampleviewergradient)');
11101152

11111153
const legendScale =
11121154
d3.scaleLinear().domain([this.minSal, this.maxSal]).range([

0 commit comments

Comments
 (0)