diff --git a/tensorboard/components/tensor_widget/colormap-test.ts b/tensorboard/components/tensor_widget/colormap-test.ts index e1fb24c9fb..2b63152c04 100644 --- a/tensorboard/components/tensor_widget/colormap-test.ts +++ b/tensorboard/components/tensor_widget/colormap-test.ts @@ -17,36 +17,40 @@ limitations under the License. import {expect} from 'chai'; -import {GrayscaleColorMap} from './colormap'; +import {GrayscaleColorMap, JetColorMap} from './colormap'; describe('GrayscaleColorMap', () => { it('max < min causes constructor error', () => { const min = 3; const max = 2; - expect(() => new GrayscaleColorMap(min, max)).to.throw(/max.*<.*min/); + expect(() => new GrayscaleColorMap({min, max})).to.throw(/max.*<.*min/); }); it('NaN or Infinity min or max causes constructor error', () => { - expect(() => new GrayscaleColorMap(0, Infinity)).to.throw( + expect(() => new GrayscaleColorMap({min: 0, max: Infinity})).to.throw( /max.*not finite/ ); - expect(() => new GrayscaleColorMap(0, -Infinity)).to.throw( + expect(() => new GrayscaleColorMap({min: 0, max: -Infinity})).to.throw( /max.*not finite/ ); - expect(() => new GrayscaleColorMap(0, NaN)).to.throw(/max.*not finite/); - expect(() => new GrayscaleColorMap(Infinity, 0)).to.throw( + expect(() => new GrayscaleColorMap({min: 0, max: NaN})).to.throw( + /max.*not finite/ + ); + expect(() => new GrayscaleColorMap({min: Infinity, max: 0})).to.throw( /min.*not finite/ ); - expect(() => new GrayscaleColorMap(-Infinity, 0)).to.throw( + expect(() => new GrayscaleColorMap({min: -Infinity, max: 0})).to.throw( + /min.*not finite/ + ); + expect(() => new GrayscaleColorMap({min: NaN, max: 0})).to.throw( /min.*not finite/ ); - expect(() => new GrayscaleColorMap(NaN, 0)).to.throw(/min.*not finite/); }); it('max > min, finite values', () => { const min = 0; const max = 10; - const colormap = new GrayscaleColorMap(min, max); + const colormap = new GrayscaleColorMap({min, max}); expect(colormap.getRGB(0)).to.eql([0, 0, 0]); expect(colormap.getRGB(5)).to.eql([127.5, 127.5, 127.5]); expect(colormap.getRGB(10)).to.eql([255, 255, 255]); @@ -58,7 +62,7 @@ describe('GrayscaleColorMap', () => { it('max > min, non-finite values', () => { const min = 0; const max = 10; - const colormap = new GrayscaleColorMap(min, max); + const colormap = new GrayscaleColorMap({min, max}); expect(colormap.getRGB(NaN)).to.eql([255, 0, 0]); expect(colormap.getRGB(-Infinity)).to.eql([255, 255 / 2, 0]); expect(colormap.getRGB(Infinity)).to.eql([0, 0, 255]); @@ -67,7 +71,7 @@ describe('GrayscaleColorMap', () => { it('max === min, non-finite values', () => { const min = -3.2; const max = -3.2; - const colormap = new GrayscaleColorMap(min, max); + const colormap = new GrayscaleColorMap({min, max}); expect(colormap.getRGB(-32)).to.eql([127.5, 127.5, 127.5]); expect(colormap.getRGB(-3.2)).to.eql([127.5, 127.5, 127.5]); expect(colormap.getRGB(0)).to.eql([127.5, 127.5, 127.5]); @@ -77,3 +81,61 @@ describe('GrayscaleColorMap', () => { expect(colormap.getRGB(Infinity)).to.eql([0, 0, 255]); }); }); + +describe('JetColormap', () => { + it('max < min causes constructor error', () => { + const min = 3; + const max = 2; + expect(() => new JetColorMap({min, max})).to.throw(/max.*<.*min/); + }); + + it('NaN or Infinity min or max causes constructor error', () => { + expect(() => new JetColorMap({min: 0, max: Infinity})).to.throw( + /max.*not finite/ + ); + expect(() => new JetColorMap({min: 0, max: -Infinity})).to.throw( + /max.*not finite/ + ); + expect(() => new JetColorMap({min: 0, max: NaN})).to.throw( + /max.*not finite/ + ); + expect(() => new JetColorMap({min: Infinity, max: 0})).to.throw( + /min.*not finite/ + ); + expect(() => new JetColorMap({min: -Infinity, max: 0})).to.throw( + /min.*not finite/ + ); + expect(() => new JetColorMap({min: NaN, max: 0})).to.throw( + /min.*not finite/ + ); + }); + + it('max > min, finite values', () => { + const min = 0; + const max = 10; + const colormap = new JetColorMap({min, max}); + expect(colormap.getRGB(0)).to.eql([0, 0, 255]); + expect(colormap.getRGB(5)).to.eql([127.5, 255, 127.5]); + expect(colormap.getRGB(10)).to.eql([255, 0, 0]); + // Over-limits. + expect(colormap.getRGB(-100)).to.eql([0, 0, 255]); + expect(colormap.getRGB(500)).to.eql([255, 0, 0]); + }); + + it('max > min, non-finite values', () => { + const min = 0; + const max = 10; + const colormap = new JetColorMap({min, max}); + expect(colormap.getRGB(NaN)).to.eql([255 * 0.25, 255 * 0.25, 255 * 0.25]); + expect(colormap.getRGB(-Infinity)).to.eql([ + 255 * 0.5, + 255 * 0.5, + 255 * 0.5, + ]); + expect(colormap.getRGB(Infinity)).to.eql([ + 255 * 0.75, + 255 * 0.75, + 255 * 0.75, + ]); + }); +}); diff --git a/tensorboard/components/tensor_widget/colormap.ts b/tensorboard/components/tensor_widget/colormap.ts index b6b3180d7a..c48705ab6f 100644 --- a/tensorboard/components/tensor_widget/colormap.ts +++ b/tensorboard/components/tensor_widget/colormap.ts @@ -17,6 +17,24 @@ limitations under the License. const MAX_RGB = 255; +/** Configuration for a colormap. */ +export interface ColorMapConfig { + /** + * Minimum value that the color map can map to without clipping. + * Must be a finite value. + */ + min: number; + + /** + * Minimum value that the color map can map to without clipping. + * Must be a finite value and be `>= min`. + * + * In the case of `max === min`, all finite values mapped to the + * midpoint of the color scale. + */ + max: number; +} + /** * Abstract base class for colormap. * @@ -28,15 +46,15 @@ export abstract class ColorMap { * @param min Minimum. Must be a finite value. * @param max Maximum. Must be finite and >= `min`. */ - constructor(protected readonly min: number, protected readonly max: number) { - if (!isFinite(min)) { - throw new Error(`min value (${min}) is not finite`); + constructor(protected config: ColorMapConfig) { + if (!isFinite(config.min)) { + throw new Error(`min value (${config.min}) is not finite`); } - if (!isFinite(max)) { - throw new Error(`max value (${max}) is not finite`); + if (!isFinite(config.max)) { + throw new Error(`max value (${config.max}) is not finite`); } - if (max < min) { - throw new Error(`max (${max}) is < min (${min})`); + if (config.max < config.min) { + throw new Error(`max (${config.max}) is < min (${config.min})`); } } @@ -47,6 +65,67 @@ export abstract class ColorMap { * The range of RGB values is [0, 255]. */ abstract getRGB(value: number): [number, number, number]; + + /** + * Render the colormap as a horizontal scale, while highlighting a specifc + * value. + * + * The value is highlighted if and only if it is `>= this.config.min` and + * `<= this.config.max`. + * + * @param canvas The canvas on which + * @param value The number to highlight (optional). + */ + render(canvas: HTMLCanvasElement, value?: number) { + if (this.config.min === this.config.max) { + return; + } + const context = canvas.getContext('2d'); + if (context == null) { + return; + } + const steps = 100; + const cellWidth = canvas.width / steps; + const height = canvas.height; + const verticalMargin = 0.2; + const barHeight = height * (1 - 2 * verticalMargin); + for (let i = 0; i < steps; ++i) { + const value = + (this.config.max - this.config.min) * (i / steps) + this.config.min; + const x = cellWidth * i; + const y = height * verticalMargin; + const [r, g, b] = this.getRGB(value); + context.beginPath(); + context.fillStyle = `rgba(${r}, ${g}, ${b}, 1)`; + context.fillRect(x, y, cellWidth, barHeight); + context.stroke(); + } + + if (value != null) { + const tickWidth = 8; + if (value >= this.config.min && value <= this.config.max) { + // Highlight the relative position of `value` along the color scale. + const tickX = + ((value - this.config.min) / (this.config.max - this.config.min)) * + canvas.width; + + // Draw the triangle on the top. + context.beginPath(); + context.fillStyle = 'rgba(0, 0, 0, 1)'; + context.moveTo(tickX, verticalMargin * height); + context.lineTo(tickX - tickWidth / 2, 0); + context.lineTo(tickX + tickWidth / 2, 0); + context.fill(); + + // Draw the triangle on the bottom. + context.beginPath(); + context.moveTo(tickX, (1 - verticalMargin) * height); + context.lineTo(tickX - tickWidth / 2, height); + context.lineTo(tickX + tickWidth / 2, height); + context.fill(); + } + } + } } /** @@ -68,13 +147,52 @@ export class GrayscaleColorMap extends ColorMap { return [MAX_RGB, MAX_RGB / 2, 0]; } } - let relativeValue = - this.min === this.max ? 0.5 : (value - this.min) / (this.max - this.min); - relativeValue = Math.max(Math.min(relativeValue, 1), 0); - return [ - MAX_RGB * relativeValue, - MAX_RGB * relativeValue, - MAX_RGB * relativeValue, - ]; + let relValue = + this.config.min === this.config.max + ? 0.5 + : (value - this.config.min) / (this.config.max - this.config.min); + relValue = Math.max(Math.min(relValue, 1), 0); + return [MAX_RGB * relValue, MAX_RGB * relValue, MAX_RGB * relValue]; + } +} + +export class JetColorMap extends ColorMap { + getRGB(value: number): [number, number, number] { + if (isNaN(value)) { + // NaN. + return [MAX_RGB * 0.25, MAX_RGB * 0.25, MAX_RGB * 0.25]; + } else if (!isFinite(value)) { + if (value < 0) { + // -Infinity. + return [MAX_RGB * 0.5, MAX_RGB * 0.5, MAX_RGB * 0.5]; + } else { + // +Infinity. + return [MAX_RGB * 0.75, MAX_RGB * 0.75, MAX_RGB * 0.75]; + } + } + + let relR = 0; + let relG = 0; + let relB = 0; + const lim0 = 0.35; + const lim1 = 0.65; + + let relValue = + this.config.min === this.config.max + ? 0.5 + : (value - this.config.min) / (this.config.max - this.config.min); + relValue = Math.max(Math.min(relValue, 1), 0); + if (relValue <= lim0) { + relG = relValue / lim0; + relB = 1; + } else if (relValue > lim0 && relValue <= lim1) { + relR = (relValue - lim0) / (lim1 - lim0); + relG = 1; + relB = (lim1 - relValue) / (lim1 - lim0); + } else if (relValue > lim1) { + relR = 1; + relG = (1 - relValue) / (1 - lim1); + } + return [relR * MAX_RGB, relG * MAX_RGB, relB * MAX_RGB]; } } diff --git a/tensorboard/components/tensor_widget/menu.ts b/tensorboard/components/tensor_widget/menu.ts index 0cb55c2c27..1ed8a19ca4 100644 --- a/tensorboard/components/tensor_widget/menu.ts +++ b/tensorboard/components/tensor_widget/menu.ts @@ -21,6 +21,13 @@ type EventCallback = (event: Event) => void | Promise; export interface MenuItemConfig { /** Caption displayed on the menu item. */ caption: string; + + /** + * A function that determines whether menu item is currently enabled. + * + * If not provided, the menu item will always be enabled. + */ + isEnabled?: () => boolean; } /** @@ -29,13 +36,6 @@ export interface MenuItemConfig { export interface SingleActionMenuItemConfig extends MenuItemConfig { /** The callback that gets called when the menu item is clicked. */ callback: EventCallback; - - /** - * A function that determines whether menu item is currently enabled. - * - * If not provided, the menu item will always be enabled. - */ - isEnabled?: () => boolean; } export interface ChoiceMenuItemConfig extends MenuItemConfig { @@ -267,12 +267,9 @@ export class Menu { // This is a single-command item. const singleActionConfig = item as SingleActionMenuItemConfig; outerItemConfig.onClick = singleActionConfig.callback; - if ( - singleActionConfig.isEnabled != null && - !singleActionConfig.isEnabled() - ) { - outerItemConfig.disabled = true; - } + } + if (item.isEnabled != null && !item.isEnabled()) { + outerItemConfig.disabled = true; } outerItemConfigs.push(outerItemConfig); }); diff --git a/tensorboard/components/tensor_widget/tensor-widget-impl.ts b/tensorboard/components/tensor_widget/tensor-widget-impl.ts index ce78dfe1b4..481f38f185 100644 --- a/tensorboard/components/tensor_widget/tensor-widget-impl.ts +++ b/tensorboard/components/tensor_widget/tensor-widget-impl.ts @@ -49,7 +49,12 @@ import { BaseTensorNumericSummary, BooleanOrNumericTensorNumericSummary, } from './health-pill-types'; -import {ColorMap, GrayscaleColorMap} from './colormap'; +import { + ColorMap, + ColorMapConfig, + GrayscaleColorMap, + JetColorMap, +} from './colormap'; const DETAILED_VALUE_ATTR_KEY = 'detailed-value'; @@ -64,6 +69,14 @@ enum ValueRenderMode { * Implementation of TensorWidget. */ +/** Color-map look-up table. */ +const colorMaps: { + [colorMapName: string]: new (config: ColorMapConfig) => ColorMap; +} = { + Grayscale: GrayscaleColorMap, + Jet: JetColorMap, +}; + /** An implementation of TensorWidget single-tensor view. */ export class TensorWidgetImpl implements TensorWidget { private readonly options: TensorWidgetOptions; @@ -109,12 +122,19 @@ export class TensorWidgetImpl implements TensorWidget { // Value render mode. protected valueRenderMode: ValueRenderMode; + // Name of color map (takes effect on IMAGE value render mode only). + protected colorMapName: string = 'Grayscale'; + protected colorMap: ColorMap | null = null; + // Whether indices should be rendered on ruler ticks on the top and left. // Determined dynamically based on the current size of the ticks. protected showIndicesOnTicks: boolean = false; // Size of each cell used to display the tensor value under the 'image' mode. protected imageCellSize = 16; + protected readonly minImageCellSize = 4; + protected readonly maxImageCellSize = 40; + protected readonly zoomStepRatio = 1.2; protected numericSummary: BaseTensorNumericSummary | null = null; @@ -291,20 +311,29 @@ export class TensorWidgetImpl implements TensorWidget { } }, } as ChoiceMenuItemConfig); - const zoomStepRatio = 1.2; + this.menuConfig.items.push({ - caption: 'Zoom in (Image mode only)', - callback: (event) => { - this.imageCellSize *= zoomStepRatio; + caption: 'Select color map...', + options: Object.keys(colorMaps), + defaultSelection: 0, + callback: (currentMode: number) => { + this.colorMapName = Object.keys(colorMaps)[currentMode]; this.renderValues(); }, isEnabled: () => this.valueRenderMode === ValueRenderMode.IMAGE, + } as ChoiceMenuItemConfig); + + this.menuConfig.items.push({ + caption: 'Zoom in (Image mode)', + callback: () => { + this.zoomInOneStepAndRenderValues(); + }, + isEnabled: () => this.valueRenderMode === ValueRenderMode.IMAGE, } as SingleActionMenuItemConfig); this.menuConfig.items.push({ - caption: 'Zoom out (Image mode only)', - callback: (event) => { - this.imageCellSize /= zoomStepRatio; - this.renderValues(); + caption: 'Zoom out (Image mode)', + callback: () => { + this.zoomOutOneStepAndRenderValues(); }, isEnabled: () => this.valueRenderMode === ValueRenderMode.IMAGE, } as SingleActionMenuItemConfig); @@ -316,6 +345,20 @@ export class TensorWidgetImpl implements TensorWidget { } } + private zoomInOneStepAndRenderValues() { + if (this.imageCellSize * this.zoomStepRatio <= this.maxImageCellSize) { + this.imageCellSize *= this.zoomStepRatio; + this.renderValues(); + } + } + + private zoomOutOneStepAndRenderValues() { + if (this.imageCellSize / this.zoomStepRatio >= this.minImageCellSize) { + this.imageCellSize /= this.zoomStepRatio; + this.renderValues(); + } + } + /** Render the thumb that when clicked, toggles the menu display state. */ private renderMenuThumb() { if (this.headerSection == null) { @@ -360,6 +403,28 @@ export class TensorWidgetImpl implements TensorWidget { this.rootElement.appendChild(this.valueSection); this.valueSection.addEventListener('wheel', async (event) => { + let zoomKeyPressed = false; + if ( + this.options.wheelZoomKey == null || + this.options.wheelZoomKey === 'ctrl' + ) { + zoomKeyPressed = event.ctrlKey; + } else if (this.options.wheelZoomKey === 'alt') { + zoomKeyPressed = event.altKey; + } else if (this.options.wheelZoomKey === 'shift') { + zoomKeyPressed = event.shiftKey; + } + if (zoomKeyPressed && this.valueRenderMode === ValueRenderMode.IMAGE) { + event.stopPropagation(); + event.preventDefault(); + if (event.deltaY > 0) { + this.zoomOutOneStepAndRenderValues(); + } else { + this.zoomInOneStepAndRenderValues(); + } + return; + } + if (this.selection == null) { return; } @@ -788,7 +853,16 @@ export class TensorWidgetImpl implements TensorWidget { 'missing minimum or maximum values in numeric summary' ); } - colorMap = new GrayscaleColorMap(minimum as number, maximum as number); + const colorMapConfig: ColorMapConfig = { + min: minimum as number, + max: maximum as number, + }; + if (this.colorMapName in colorMaps) { + this.colorMap = new colorMaps[this.colorMapName](colorMapConfig); + } else { + // Color-map name is not found. Use the default: Grayscale colormap. + this.colorMap = new GrayscaleColorMap(colorMapConfig); + } } for (let i = 0; i < numRows; ++i) { @@ -800,7 +874,7 @@ export class TensorWidgetImpl implements TensorWidget { ) { const value = (values as number[][] | boolean[][] | string[][])[i][j]; if (valueRenderMode === ValueRenderMode.IMAGE) { - const [red, green, blue] = (colorMap as ColorMap).getRGB( + const [red, green, blue] = (this.colorMap as ColorMap).getRGB( value as number ); valueDiv.style.backgroundColor = `rgb(${red}, ${green}, ${blue})`; @@ -988,6 +1062,18 @@ export class TensorWidgetImpl implements TensorWidget { this.valueTooltip.style.top = `${top}px`; this.valueTooltip.style.left = `${left}px`; this.valueTooltip.style.display = 'block'; + + // If the current render mode is IMAGE, show the color bar and + // indicate the position of the current element along the color-bar scale. + if ( + this.valueRenderMode == ValueRenderMode.IMAGE && + this.colorMap != null + ) { + const colorBarCanvas = document.createElement('canvas'); + colorBarCanvas.classList.add('tensor-widget-value-tooltip-colorbar'); + this.valueTooltip.appendChild(colorBarCanvas); + this.colorMap.render(colorBarCanvas, parseFloat(detailedValueString)); + } } private hideValueTooltip() { diff --git a/tensorboard/components/tensor_widget/tensor-widget.css b/tensorboard/components/tensor_widget/tensor-widget.css index 52bf58786c..dfc6dd7b25 100644 --- a/tensorboard/components/tensor_widget/tensor-widget.css +++ b/tensorboard/components/tensor_widget/tensor-widget.css @@ -193,6 +193,13 @@ limitations under the License. font-size: 13px; padding: 5px; position: absolute; + user-select: none; + width: 240px; +} + +.tensor-widget-value-tooltip-colorbar { + height: 24px; + width: 95%; } .tensor-widget-value-tooltip-indices { diff --git a/tensorboard/components/tensor_widget/types.ts b/tensorboard/components/tensor_widget/types.ts index a6eb7cbe18..7a53d4d09f 100644 --- a/tensorboard/components/tensor_widget/types.ts +++ b/tensorboard/components/tensor_widget/types.ts @@ -185,6 +185,14 @@ export interface TensorWidgetOptions { */ decimalPlaces?: number; + /** + * Whether to use the Alt, Ctrl or Shift key with the mouse for zooming under + * the image value-rendering mode. + * + * Defaults to Ctrl key ('ctrl'). + */ + wheelZoomKey?: 'alt' | 'ctrl' | 'shift'; + /** TODO(cais): Add support for custom tensor renderers. */ } diff --git a/tensorboard/plugins/debugger/tf_debugger_dashboard/tf-tensor-value-view.html b/tensorboard/plugins/debugger/tf_debugger_dashboard/tf-tensor-value-view.html index be79d4c729..61642ce8ec 100644 --- a/tensorboard/plugins/debugger/tf_debugger_dashboard/tf-tensor-value-view.html +++ b/tensorboard/plugins/debugger/tf_debugger_dashboard/tf-tensor-value-view.html @@ -342,7 +342,8 @@ if (this.tensorWidget == null) { this.tensorWidget = tensor_widget.tensorWidget( this.$$('#tensor-widget'), - tensorView + tensorView, + {wheelZoomKey: 'alt'} ); } this.tensorWidget.render();