From cf1599c0a13b5cae32d66dfaa1028c12ccb46ba9 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Wed, 2 Oct 2019 17:34:50 -0400 Subject: [PATCH 01/16] [tensor-widget] Add Menu --- tensorboard/components/tensor_widget/BUILD | 1 + tensorboard/components/tensor_widget/menu.ts | 55 ++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 tensorboard/components/tensor_widget/menu.ts diff --git a/tensorboard/components/tensor_widget/BUILD b/tensorboard/components/tensor_widget/BUILD index e2e04161d9..34c3c1072e 100644 --- a/tensorboard/components/tensor_widget/BUILD +++ b/tensorboard/components/tensor_widget/BUILD @@ -47,6 +47,7 @@ tf_ts_library( srcs = [ "dtype-utils.ts", "health-pill-types.ts", + "menu.ts", "selection.ts", "shape-utils.ts", "slicing-control.ts", diff --git a/tensorboard/components/tensor_widget/menu.ts b/tensorboard/components/tensor_widget/menu.ts new file mode 100644 index 0000000000..e0c7e5abc5 --- /dev/null +++ b/tensorboard/components/tensor_widget/menu.ts @@ -0,0 +1,55 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +export interface MenuItemConfig { + /** Caption displayed on the menu item. */ + caption: string; + + /** Click callback for the menu item. */ + onClick: () => void | Promise; +} + +export interface MenuConfig { + /** An ordered list of items that comprise the menu. */ + items: MenuItemConfig[]; +} + +/** + * A class for menu that supports configurable items and callbacks. + */ +export class Menu { + /** + * Constructor for the Menu class. + * + * @param config Configuration for the menu. + */ + constructor(private readonly config: MenuConfig) { + } + + /** + * Show the menu. + * + * @param top The top coordinate for the top-left corner of the menu. + * @param left The left coordinate for the top-left corner of the menu. + */ + show(top: number, left: number) { + + } + + /** Hide the menu. */ + hide() { + + } +} \ No newline at end of file From 6524ccb71e70cc0fc766da641706f05fbf161766 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Thu, 3 Oct 2019 23:47:19 -0400 Subject: [PATCH 02/16] save --- tensorboard/components/tensor_widget/menu.ts | 140 +++++++++++++++++- .../tensor_widget/tensor-widget-impl.ts | 64 +++++++- .../tensor_widget/tensor-widget.css | 15 ++ 3 files changed, 213 insertions(+), 6 deletions(-) diff --git a/tensorboard/components/tensor_widget/menu.ts b/tensorboard/components/tensor_widget/menu.ts index e0c7e5abc5..d9564c9199 100644 --- a/tensorboard/components/tensor_widget/menu.ts +++ b/tensorboard/components/tensor_widget/menu.ts @@ -13,12 +13,55 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +type Callback = () => void | Promise; +type HoverCallback = (event: Event) => void | Promise; + +/** + * The base interface for a menu item. + */ export interface MenuItemConfig { /** Caption displayed on the menu item. */ caption: string; +} - /** Click callback for the menu item. */ - onClick: () => void | Promise; +/** + * A menu item which when clicked, triggers a single action. + */ +export interface SingleActionMenuItemConfig extends MenuItemConfig { + /** The callback that gets called when the menu item is clicked. */ + callback: Callback; +} + +export interface ChoiceMenuItemConfig extends MenuItemConfig { + /** All possible options. */ + options: string[]; + + /** The default selection from `options`, a 0-based index. */ + defaultSelection: number; + + /** + * The callback that gets called when the selection has changed. + * + * The `currentSelection` argument is the 0-based index for the currently + * selected option. + */ + callback: (currentSelection: number) => void | Promise; +} + +/** + * A menu item that supports a binary toggle state. + */ +export interface ToggleMenuItemConfig extends MenuItemConfig { + /** The default state of the menu item. */ + defaultState: boolean; + + /** + * The callback that gets called when the toggle state has changed. + * + * The `currentState` argument is a boolean indicating whether the + * toggle menu item is activated (`true`) or not (`false`). + */ + callback: (currentState: boolean) => void | Promise; } export interface MenuConfig { @@ -26,16 +69,78 @@ export interface MenuConfig { items: MenuItemConfig[]; } +/** + * Helper class: A menu item without hierarchy. + */ +class FlatMenu { + private isShown: boolean = false; + private dropdown: HTMLDivElement; + // TODO(cais): Tie in the arg lengths. + constructor(parentElement: HTMLDivElement) { + this.dropdown = document.createElement('div'); + this.dropdown.classList.add('tensor-widget-dim-dropdown'); + this.dropdown.style.position = 'fixed'; + this.dropdown.style.display = 'none'; + parentElement.appendChild(this.dropdown); + } + + show( + top: number, + left: number, + captions: string[], + onHoverCallbacks: Array + ) { + captions.forEach((caption, i) => { + const menuItem = document.createElement('div'); + menuItem.classList.add('tensor-widget-dim-dropdown-menu-item'); + menuItem.textContent = caption; + this.dropdown.appendChild(menuItem); + const onHover = onHoverCallbacks[i]; + if (onHover !== null) { + menuItem.addEventListener('mouseover', onHover); + // TODO(cais): Add mouseexit callback. + } + }); + this.dropdown.style.display = 'block'; + this.dropdown.style.top = top + 'px'; + this.dropdown.style.left = left + 'px'; + const actualRect = this.dropdown.getBoundingClientRect(); + const topOffset = actualRect.top - top; + const leftOffset = actualRect.left - left; + this.dropdown.style.top = (top - topOffset).toFixed(1) + 'px'; + this.dropdown.style.left = (left - leftOffset).toFixed(1) + 'px'; + this.isShown = true; + } + + hide() { + this.dropdown.style.display = 'none'; + while (this.dropdown.firstChild) { + this.dropdown.removeChild(this.dropdown.firstChild); + } + this.isShown = false; + } + + shown() { + return this.isShown; + } +} + /** * A class for menu that supports configurable items and callbacks. */ export class Menu { + private baseFlatMenu: FlatMenu; + /** * Constructor for the Menu class. * * @param config Configuration for the menu. */ - constructor(private readonly config: MenuConfig) { + constructor( + private readonly config: MenuConfig, + private readonly parentElement: HTMLDivElement + ) { + this.baseFlatMenu = new FlatMenu(parentElement); } /** @@ -45,11 +150,36 @@ export class Menu { * @param left The left coordinate for the top-left corner of the menu. */ show(top: number, left: number) { - + const captions: string[] = this.config.items.map((item) => item.caption); + const onHovers: Array = this.config.items.map( + (item) => { + if ((item as ChoiceMenuItemConfig).options != null) { + // TODO(cais): Check to make sure it's not empty? + return (event) => { + const captions = (item as ChoiceMenuItemConfig).options; + const optionsFlatMenu = new FlatMenu(this.parentElement); + console.log(captions); + console.log(event.srcElement); + const box = (event.srcElement as HTMLDivElement).getBoundingClientRect(); + const top = box.top; + const left = box.right; + const onHovers = captions.map((caption) => null); + optionsFlatMenu.show(top, left, captions, onHovers); + }; + } else { + return null; + } + } + ); + this.baseFlatMenu.show(top, left, captions, onHovers); } /** Hide the menu. */ hide() { + this.baseFlatMenu.hide(); + } + shown(): boolean { + return this.baseFlatMenu.shown(); } -} \ No newline at end of file +} diff --git a/tensorboard/components/tensor_widget/tensor-widget-impl.ts b/tensorboard/components/tensor_widget/tensor-widget-impl.ts index 104ec70e6a..b20262e215 100644 --- a/tensorboard/components/tensor_widget/tensor-widget-impl.ts +++ b/tensorboard/components/tensor_widget/tensor-widget-impl.ts @@ -19,6 +19,7 @@ import { isIntegerDType, isStringDType, } from './dtype-utils'; +import {ChoiceMenuItemConfig, Menu, MenuConfig} from './menu'; import {TensorElementSelection} from './selection'; import { formatShapeForDisplay, @@ -56,6 +57,8 @@ export class TensorWidgetImpl implements TensorWidget { // Constituent UI elements. protected headerSection: HTMLDivElement | null = null; protected infoSubsection: HTMLDivElement | null = null; + protected menuThumb: HTMLDivElement | null = null; + protected slicingSpecRoot: HTMLDivElement | null = null; protected valueSection: HTMLDivElement | null = null; protected topRuler: HTMLDivElement | null = null; @@ -83,6 +86,9 @@ export class TensorWidgetImpl implements TensorWidget { // Element selection. protected selection: TensorElementSelection | null = null; + // Menu configuration. + protected menuConfig: MenuConfig | null = null; + constructor( private readonly rootElement: HTMLDivElement, private readonly tensorView: TensorView, @@ -140,7 +146,7 @@ export class TensorWidgetImpl implements TensorWidget { } this.renderInfo(); // TODO(cais): Implement and call renderHealthPill(). - // TODO(cais): Implement and call createMenu(); + this.createMenu(); } /** @@ -230,6 +236,62 @@ export class TensorWidgetImpl implements TensorWidget { this.infoSubsection.appendChild(shapeTagDiv); } + private createMenu() { + this.menuConfig = {items: []}; + if ( + isFloatDType(this.tensorView.spec.dtype) || + isIntegerDType(this.tensorView.spec.dtype) || + isBooleanDType(this.tensorView.spec.dtype) + ) { + this.menuConfig.items.push({ + caption: 'Select display mode...', + options: ['Text', 'Image'], + defaultSelection: 0, + callback: (currentMode: number) => { + console.log(`Display mode changed: ${currentMode}`); + }, + } as ChoiceMenuItemConfig); + } + if (this.menuConfig !== null) { + this.menu = new Menu(this.menuConfig, this + .headerSection as HTMLDivElement); + } + this.renderMenuThumb(); + } + + private menu: Menu | null = null; + + private dropdown: HTMLDivElement | null = null; // TODO(cais): Move to top; + + /** Render the thumb that when clicked, toggles the menu display state. */ + private renderMenuThumb() { + if (this.headerSection == null) { + throw new Error( + 'Rendering menu thumb failed due to missing header section.' + ); + } + this.menuThumb = document.createElement('div'); + this.menuThumb.textContent = '⋮'; + this.menuThumb.classList.add('tensor-widget-menu-thumb'); + this.headerSection.appendChild(this.menuThumb); + + // let menuShown = false; // TODO(cais): Make a class member? + this.menuThumb.addEventListener('click', () => { + if (this.menu === null) { + return; + } + if (this.menu.shown()) { + this.menu.hide(); + } else { + const rect = (this.menuThumb as HTMLDivElement).getBoundingClientRect(); + const top = rect.bottom; + const left = rect.left; + console.log(`top = ${top}, left = ${left}`); + this.menu.show(top, left); + } + }); + } + /** * Fill in the content of the value divs given the current slicing spec. */ diff --git a/tensorboard/components/tensor_widget/tensor-widget.css b/tensorboard/components/tensor_widget/tensor-widget.css index 5db4db8992..efbb17a191 100644 --- a/tensorboard/components/tensor_widget/tensor-widget.css +++ b/tensorboard/components/tensor_widget/tensor-widget.css @@ -110,6 +110,21 @@ limitations under the License. vertical-align: middle; } +.tensor-widget-menu-thumb { + color: rgb(32, 33, 36); + cursor: pointer; + font-weight: bold; + font-size: 16px; + float: right; + left: -10px; + position: relative; + user-select: none; +} + +.tensor-widget-menu-thumb:hover { + color: rgb(227, 116, 0); +} + .tensor-widget-shape { color: rgb(60, 60, 60); display: inline-block; From 5edecba01a394943f5ed63d6c26678f56acaca99 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 4 Oct 2019 11:15:01 -0400 Subject: [PATCH 03/16] save --- tensorboard/components/tensor_widget/menu.ts | 110 +++++++++++++++--- .../tensor_widget/tensor-widget-impl.ts | 1 - .../tensor_widget/tensor-widget.css | 2 +- 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/tensorboard/components/tensor_widget/menu.ts b/tensorboard/components/tensor_widget/menu.ts index d9564c9199..b10bcced97 100644 --- a/tensorboard/components/tensor_widget/menu.ts +++ b/tensorboard/components/tensor_widget/menu.ts @@ -14,7 +14,7 @@ limitations under the License. ==============================================================================*/ type Callback = () => void | Promise; -type HoverCallback = (event: Event) => void | Promise; +type EventCallback = (event: Event) => void | Promise; /** * The base interface for a menu item. @@ -69,6 +69,13 @@ export interface MenuConfig { items: MenuItemConfig[]; } + +interface FlatMenuItemConfig { + caption: string, + onClick: EventCallback | null; + onHover: EventCallback | null; +} + /** * Helper class: A menu item without hierarchy. */ @@ -81,6 +88,9 @@ class FlatMenu { this.dropdown.classList.add('tensor-widget-dim-dropdown'); this.dropdown.style.position = 'fixed'; this.dropdown.style.display = 'none'; + this.dropdown.addEventListener('mouseleave', () => { + this.hide(); + }); parentElement.appendChild(this.dropdown); } @@ -88,18 +98,44 @@ class FlatMenu { top: number, left: number, captions: string[], - onHoverCallbacks: Array + onClickCallbacks: Array, + onHoverCallbacks: Array ) { captions.forEach((caption, i) => { const menuItem = document.createElement('div'); menuItem.classList.add('tensor-widget-dim-dropdown-menu-item'); menuItem.textContent = caption; this.dropdown.appendChild(menuItem); + const onClick = onClickCallbacks[i]; const onHover = onHoverCallbacks[i]; - if (onHover !== null) { - menuItem.addEventListener('mouseover', onHover); - // TODO(cais): Add mouseexit callback. - } + menuItem.addEventListener('click', (event) => { + if (onClick !== null) { + onClick(event); + } + this.hide(); + }); + menuItem.addEventListener('mouseenter', (event) => { + if (onHover !== null) { + onHover(event); + } + menuItem.classList.add('tensor-widget-dim-dropdown-menu-item-active'); + }); + menuItem.addEventListener('mouseleave', () => { + menuItem.classList.remove( + 'tensor-widget-dim-dropdown-menu-item-active' + ); + if (onHover === null) { + return; + } + const childrenToRemove: Element[] = []; + for (let i = 0; i < menuItem.children.length; ++i) { + const child = menuItem.children[i]; + if (child.classList.contains('tensor-widget-dim-dropdown')) { + childrenToRemove.push(child); + } + } + childrenToRemove.forEach((child) => menuItem.removeChild(child)); + }); }); this.dropdown.style.display = 'block'; this.dropdown.style.top = top + 'px'; @@ -131,6 +167,9 @@ class FlatMenu { export class Menu { private baseFlatMenu: FlatMenu; + // The currently selected indices for all multiple-choice menu items. + private currentChoiceSelections: {[itemIndex: number]: number}; + /** * Constructor for the Menu class. * @@ -140,7 +179,16 @@ export class Menu { private readonly config: MenuConfig, private readonly parentElement: HTMLDivElement ) { - this.baseFlatMenu = new FlatMenu(parentElement); + this.baseFlatMenu = new FlatMenu(this.parentElement); + + this.currentChoiceSelections = {}; + this.config.items.forEach((item, i) => { + if ((item as ChoiceMenuItemConfig).options != null) { + this.currentChoiceSelections[ + i + ] = (item as ChoiceMenuItemConfig).defaultSelection; + } + }); } /** @@ -151,27 +199,55 @@ export class Menu { */ show(top: number, left: number) { const captions: string[] = this.config.items.map((item) => item.caption); - const onHovers: Array = this.config.items.map( - (item) => { + const clickCallbacks: Array = this.config.items.map( + (item, i) => { + if ((item as ChoiceMenuItemConfig).options != null) { + // This is a multiple-choice item. + return null; + } else if ((item as ToggleMenuItemConfig).defaultState != null) { + // This is a binary toggle item. + // TODO(cais): Modify state. + return null; + } else { + // This is a single-command item. + return (item as SingleActionMenuItemConfig).callback; + } + } + ); + const hoverCallbacks: Array = this.config.items.map( + (item, i) => { if ((item as ChoiceMenuItemConfig).options != null) { // TODO(cais): Check to make sure it's not empty? + const currentSelectionIndex = this.currentChoiceSelections[i]; return (event) => { - const captions = (item as ChoiceMenuItemConfig).options; - const optionsFlatMenu = new FlatMenu(this.parentElement); - console.log(captions); - console.log(event.srcElement); - const box = (event.srcElement as HTMLDivElement).getBoundingClientRect(); + const parent = event.target as HTMLDivElement; + const choiceConfig = item as ChoiceMenuItemConfig; + const captions = choiceConfig.options.map((option, k) => { + return k === currentSelectionIndex ? option + ' (✓)' : option; + }); + const optionsFlatMenu = new FlatMenu(parent); + const onClicks: Array = choiceConfig.options.map( + (option, k) => { + return () => { + if (currentSelectionIndex !== k) { + this.currentChoiceSelections[i] = k; + choiceConfig.callback(k); + } + }; + } + ); + const onHovers = captions.map(() => null); + const box = parent.getBoundingClientRect(); const top = box.top; const left = box.right; - const onHovers = captions.map((caption) => null); - optionsFlatMenu.show(top, left, captions, onHovers); + optionsFlatMenu.show(top, left, captions, onClicks, onHovers); }; } else { return null; } } ); - this.baseFlatMenu.show(top, left, captions, onHovers); + this.baseFlatMenu.show(top, left, captions, clickCallbacks, hoverCallbacks); } /** Hide the menu. */ diff --git a/tensorboard/components/tensor_widget/tensor-widget-impl.ts b/tensorboard/components/tensor_widget/tensor-widget-impl.ts index b20262e215..8eb860904b 100644 --- a/tensorboard/components/tensor_widget/tensor-widget-impl.ts +++ b/tensorboard/components/tensor_widget/tensor-widget-impl.ts @@ -286,7 +286,6 @@ export class TensorWidgetImpl implements TensorWidget { const rect = (this.menuThumb as HTMLDivElement).getBoundingClientRect(); const top = rect.bottom; const left = rect.left; - console.log(`top = ${top}, left = ${left}`); this.menu.show(top, left); } }); diff --git a/tensorboard/components/tensor_widget/tensor-widget.css b/tensorboard/components/tensor_widget/tensor-widget.css index efbb17a191..5d860418db 100644 --- a/tensorboard/components/tensor_widget/tensor-widget.css +++ b/tensorboard/components/tensor_widget/tensor-widget.css @@ -61,11 +61,11 @@ limitations under the License. border-bottom: 1px solid rgb(180, 180, 180); font-size: 12px; padding: 3px; + user-select: none; } .tensor-widget-dim-dropdown-menu-item-active { background-color: rgb(100, 180, 255); - color: rgb(255, 255, 255); } .tensor-widget-dtype { From e638715ce63936e8cfae302cb4e5789804545fc2 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 4 Oct 2019 11:29:16 -0400 Subject: [PATCH 04/16] save --- tensorboard/components/tensor_widget/menu.ts | 122 +++++++----------- .../tensor_widget/tensor-widget-impl.ts | 2 +- .../tensor_widget/tensor-widget.css | 4 +- 3 files changed, 49 insertions(+), 79 deletions(-) diff --git a/tensorboard/components/tensor_widget/menu.ts b/tensorboard/components/tensor_widget/menu.ts index b10bcced97..26b1afe0ac 100644 --- a/tensorboard/components/tensor_widget/menu.ts +++ b/tensorboard/components/tensor_widget/menu.ts @@ -48,22 +48,6 @@ export interface ChoiceMenuItemConfig extends MenuItemConfig { callback: (currentSelection: number) => void | Promise; } -/** - * A menu item that supports a binary toggle state. - */ -export interface ToggleMenuItemConfig extends MenuItemConfig { - /** The default state of the menu item. */ - defaultState: boolean; - - /** - * The callback that gets called when the toggle state has changed. - * - * The `currentState` argument is a boolean indicating whether the - * toggle menu item is activated (`true`) or not (`false`). - */ - callback: (currentState: boolean) => void | Promise; -} - export interface MenuConfig { /** An ordered list of items that comprise the menu. */ items: MenuItemConfig[]; @@ -97,26 +81,22 @@ class FlatMenu { show( top: number, left: number, - captions: string[], - onClickCallbacks: Array, - onHoverCallbacks: Array + itemConfigs: FlatMenuItemConfig[] ) { - captions.forEach((caption, i) => { + itemConfigs.forEach((itemConfig, i) => { const menuItem = document.createElement('div'); menuItem.classList.add('tensor-widget-dim-dropdown-menu-item'); - menuItem.textContent = caption; + menuItem.textContent = itemConfig.caption; this.dropdown.appendChild(menuItem); - const onClick = onClickCallbacks[i]; - const onHover = onHoverCallbacks[i]; menuItem.addEventListener('click', (event) => { - if (onClick !== null) { - onClick(event); + if (itemConfig.onClick !== null) { + itemConfig.onClick(event); } this.hide(); }); menuItem.addEventListener('mouseenter', (event) => { - if (onHover !== null) { - onHover(event); + if (itemConfig.onHover !== null) { + itemConfig.onHover(event); } menuItem.classList.add('tensor-widget-dim-dropdown-menu-item-active'); }); @@ -124,7 +104,7 @@ class FlatMenu { menuItem.classList.remove( 'tensor-widget-dim-dropdown-menu-item-active' ); - if (onHover === null) { + if (itemConfig.onHover === null) { return; } const childrenToRemove: Element[] = []; @@ -198,56 +178,46 @@ export class Menu { * @param left The left coordinate for the top-left corner of the menu. */ show(top: number, left: number) { - const captions: string[] = this.config.items.map((item) => item.caption); - const clickCallbacks: Array = this.config.items.map( - (item, i) => { - if ((item as ChoiceMenuItemConfig).options != null) { - // This is a multiple-choice item. - return null; - } else if ((item as ToggleMenuItemConfig).defaultState != null) { - // This is a binary toggle item. - // TODO(cais): Modify state. - return null; - } else { - // This is a single-command item. - return (item as SingleActionMenuItemConfig).callback; - } - } - ); - const hoverCallbacks: Array = this.config.items.map( - (item, i) => { - if ((item as ChoiceMenuItemConfig).options != null) { - // TODO(cais): Check to make sure it's not empty? - const currentSelectionIndex = this.currentChoiceSelections[i]; - return (event) => { - const parent = event.target as HTMLDivElement; - const choiceConfig = item as ChoiceMenuItemConfig; - const captions = choiceConfig.options.map((option, k) => { - return k === currentSelectionIndex ? option + ' (✓)' : option; + const outerItemConfigs: FlatMenuItemConfig[] = []; + + this.config.items.forEach((item, i) => { + const outerItemConfig: FlatMenuItemConfig = { + caption: item.caption, + onClick: null, + onHover: null, + }; + if ((item as ChoiceMenuItemConfig).options != null) { + // This is a multiple choice item. + const currentSelectionIndex = this.currentChoiceSelections[i]; + outerItemConfig.onHover = (event) => { + const parent = event.target as HTMLDivElement; + const choiceConfig = item as ChoiceMenuItemConfig; + const itemConfigs: FlatMenuItemConfig[] = []; + choiceConfig.options.forEach((option, k) => { + itemConfigs.push({ + caption: k === currentSelectionIndex ? option + ' (✓)' : option, + onClick: () => { + if (currentSelectionIndex !== k) { + this.currentChoiceSelections[i] = k; + choiceConfig.callback(k); + } + }, + onHover: null }); - const optionsFlatMenu = new FlatMenu(parent); - const onClicks: Array = choiceConfig.options.map( - (option, k) => { - return () => { - if (currentSelectionIndex !== k) { - this.currentChoiceSelections[i] = k; - choiceConfig.callback(k); - } - }; - } - ); - const onHovers = captions.map(() => null); - const box = parent.getBoundingClientRect(); - const top = box.top; - const left = box.right; - optionsFlatMenu.show(top, left, captions, onClicks, onHovers); - }; - } else { - return null; - } + }); + const optionsFlatMenu = new FlatMenu(parent); + const box = parent.getBoundingClientRect(); + const top = box.top; + const left = box.right; + optionsFlatMenu.show(top, left, itemConfigs); + }; + } else { + // This is a single-command item. + outerItemConfig.onClick = (item as SingleActionMenuItemConfig).callback; } - ); - this.baseFlatMenu.show(top, left, captions, clickCallbacks, hoverCallbacks); + outerItemConfigs.push(outerItemConfig); + }); + this.baseFlatMenu.show(top, left, outerItemConfigs); } /** Hide the menu. */ diff --git a/tensorboard/components/tensor_widget/tensor-widget-impl.ts b/tensorboard/components/tensor_widget/tensor-widget-impl.ts index 8eb860904b..bdaeff9ac4 100644 --- a/tensorboard/components/tensor_widget/tensor-widget-impl.ts +++ b/tensorboard/components/tensor_widget/tensor-widget-impl.ts @@ -144,9 +144,9 @@ export class TensorWidgetImpl implements TensorWidget { this.headerSection.classList.add('tensor-widget-header'); this.rootElement.appendChild(this.headerSection); } + this.createMenu(); this.renderInfo(); // TODO(cais): Implement and call renderHealthPill(). - this.createMenu(); } /** diff --git a/tensorboard/components/tensor_widget/tensor-widget.css b/tensorboard/components/tensor_widget/tensor-widget.css index 5d860418db..d3354c35f3 100644 --- a/tensorboard/components/tensor_widget/tensor-widget.css +++ b/tensorboard/components/tensor_widget/tensor-widget.css @@ -113,10 +113,10 @@ limitations under the License. .tensor-widget-menu-thumb { color: rgb(32, 33, 36); cursor: pointer; + display: inline-block; font-weight: bold; font-size: 16px; - float: right; - left: -10px; + margin-left: 10px; position: relative; user-select: none; } From d721c65ea222301fce6cdfddbc5aa1bec4964f1f Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 4 Oct 2019 14:26:40 -0400 Subject: [PATCH 05/16] Added colormap --- tensorboard/components/tensor_widget/BUILD | 2 + .../components/tensor_widget/colormap-test.ts | 79 ++++++++++++++++++ .../components/tensor_widget/colormap.ts | 59 +++++++++++++ .../components/tensor_widget/demo/demo.ts | 21 ++++- .../tensor_widget/health-pill-types.ts | 13 ++- tensorboard/components/tensor_widget/menu.ts | 14 +--- .../tensor_widget/tensor-widget-impl.ts | 83 ++++++++++++++----- .../tensor_widget/tensor-widget.css | 1 + tensorboard/components/tensor_widget/types.ts | 4 +- 9 files changed, 236 insertions(+), 40 deletions(-) create mode 100644 tensorboard/components/tensor_widget/colormap-test.ts create mode 100644 tensorboard/components/tensor_widget/colormap.ts diff --git a/tensorboard/components/tensor_widget/BUILD b/tensorboard/components/tensor_widget/BUILD index 34c3c1072e..6ebd422bd5 100644 --- a/tensorboard/components/tensor_widget/BUILD +++ b/tensorboard/components/tensor_widget/BUILD @@ -45,6 +45,7 @@ tf_ts_library( tf_ts_library( name = "tensor_widget_lib", srcs = [ + "colormap.ts", "dtype-utils.ts", "health-pill-types.ts", "menu.ts", @@ -64,6 +65,7 @@ tf_ts_library( name = "test_lib", testonly = True, srcs = [ + "colormap-test.ts", "dtype-utils-test.ts", "selection-test.ts", "shape-utils-test.ts", diff --git a/tensorboard/components/tensor_widget/colormap-test.ts b/tensorboard/components/tensor_widget/colormap-test.ts new file mode 100644 index 0000000000..e1fb24c9fb --- /dev/null +++ b/tensorboard/components/tensor_widget/colormap-test.ts @@ -0,0 +1,79 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/** Unit tests for colormaps. */ + +import {expect} from 'chai'; + +import {GrayscaleColorMap} from './colormap'; + +describe('GrayscaleColorMap', () => { + it('max < min causes constructor error', () => { + const min = 3; + const max = 2; + expect(() => new GrayscaleColorMap(min, max)).to.throw(/max.*<.*min/); + }); + + it('NaN or Infinity min or max causes constructor error', () => { + expect(() => new GrayscaleColorMap(0, Infinity)).to.throw( + /max.*not finite/ + ); + expect(() => new GrayscaleColorMap(0, -Infinity)).to.throw( + /max.*not finite/ + ); + expect(() => new GrayscaleColorMap(0, NaN)).to.throw(/max.*not finite/); + expect(() => new GrayscaleColorMap(Infinity, 0)).to.throw( + /min.*not finite/ + ); + expect(() => new GrayscaleColorMap(-Infinity, 0)).to.throw( + /min.*not finite/ + ); + expect(() => new GrayscaleColorMap(NaN, 0)).to.throw(/min.*not finite/); + }); + + it('max > min, finite values', () => { + const min = 0; + const max = 10; + const colormap = new GrayscaleColorMap(min, max); + expect(colormap.getRGB(0)).to.eql([0, 0, 0]); + expect(colormap.getRGB(5)).to.eql([127.5, 127.5, 127.5]); + expect(colormap.getRGB(10)).to.eql([255, 255, 255]); + // Over-limits. + expect(colormap.getRGB(-100)).to.eql([0, 0, 0]); + expect(colormap.getRGB(500)).to.eql([255, 255, 255]); + }); + + it('max > min, non-finite values', () => { + const min = 0; + const max = 10; + const colormap = new GrayscaleColorMap(min, max); + expect(colormap.getRGB(NaN)).to.eql([255, 0, 0]); + expect(colormap.getRGB(-Infinity)).to.eql([255, 255 / 2, 0]); + expect(colormap.getRGB(Infinity)).to.eql([0, 0, 255]); + }); + + it('max === min, non-finite values', () => { + const min = -3.2; + const max = -3.2; + const colormap = new GrayscaleColorMap(min, max); + expect(colormap.getRGB(-32)).to.eql([127.5, 127.5, 127.5]); + expect(colormap.getRGB(-3.2)).to.eql([127.5, 127.5, 127.5]); + expect(colormap.getRGB(0)).to.eql([127.5, 127.5, 127.5]); + expect(colormap.getRGB(32)).to.eql([127.5, 127.5, 127.5]); + expect(colormap.getRGB(NaN)).to.eql([255, 0, 0]); + expect(colormap.getRGB(-Infinity)).to.eql([255, 255 / 2, 0]); + expect(colormap.getRGB(Infinity)).to.eql([0, 0, 255]); + }); +}); diff --git a/tensorboard/components/tensor_widget/colormap.ts b/tensorboard/components/tensor_widget/colormap.ts new file mode 100644 index 0000000000..728054984a --- /dev/null +++ b/tensorboard/components/tensor_widget/colormap.ts @@ -0,0 +1,59 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +/** Colormap used to display numeric values */ + +const MAX_RGB = 255; + +export abstract class ColorMap { + constructor(protected readonly min: number, protected readonly max: number) { + if (!isFinite(min)) { + throw new Error(`min value (${min}) is not finite`); + } + if (!isFinite(max)) { + throw new Error(`max value (${max}) is not finite`); + } + if (max < min) { + throw new Error(`max (${max}) is < min (${min})`); + } + } + + abstract getRGB(value: number): [number, number, number]; +} + +export class GrayscaleColorMap extends ColorMap { + getRGB(value: number): [number, number, number] { + if (isNaN(value)) { + // NaN. + return [MAX_RGB, 0, 0]; + } else if (!isFinite(value)) { + if (value > 0) { + // +Infinity. + return [0, 0, MAX_RGB]; + } else { + // -Infinity. + return [MAX_RGB, MAX_RGB / 2, 0]; + } + } + let relativeValue = + this.min === this.max ? 0.5 : (value - this.min) / (this.max - this.min); + relativeValue = Math.max(Math.min(relativeValue, 1), 0); + return [ + MAX_RGB * relativeValue, + MAX_RGB * relativeValue, + MAX_RGB * relativeValue, + ]; + } +} diff --git a/tensorboard/components/tensor_widget/demo/demo.ts b/tensorboard/components/tensor_widget/demo/demo.ts index a79e3fa7f2..bd8d9ce358 100644 --- a/tensorboard/components/tensor_widget/demo/demo.ts +++ b/tensorboard/components/tensor_widget/demo/demo.ts @@ -15,6 +15,10 @@ limitations under the License. import * as tensorWidget from '../tensor-widget'; import {TensorViewSlicingSpec, TensorView} from '../types'; +import { + BaseTensorNumericSummary, + BooleanOrNumericTensorNumericSummary, +} from '../health-pill-types'; // TODO(cais): Find a way to import tfjs-core here, instead of depending on // a global variable. @@ -171,9 +175,20 @@ export function tensorToTensorView(x: any): tensorWidget.TensorView { // TODO(cais): Check memory leak. return retval; }, - getHealthPill: async () => { - // return calculateHealthPill(x); - throw new Error('Not implemented yet.'); + getNumericSummary: async () => { + const numericSummary: BaseTensorNumericSummary = { + elementCount: x.size(), + }; + if (x.dtype === 'float32' || x.dtype === 'int32') { + (numericSummary as BooleanOrNumericTensorNumericSummary).minimum = x + .min() + .dataSync(); + (numericSummary as BooleanOrNumericTensorNumericSummary).maximum = x + .max() + .dataSync(); + } + // TODO(cais): Take care of boolean dtype. + return numericSummary; }, }; } diff --git a/tensorboard/components/tensor_widget/health-pill-types.ts b/tensorboard/components/tensor_widget/health-pill-types.ts index bf8f5643b2..4b941d70c5 100644 --- a/tensorboard/components/tensor_widget/health-pill-types.ts +++ b/tensorboard/components/tensor_widget/health-pill-types.ts @@ -14,7 +14,7 @@ limitations under the License. ==============================================================================*/ /** - * Health pill: A summary of a tensor's element values. + * A summary of a tensor's element values. * * It contains information such as * - the distribution of the tensor's values among different value categories @@ -25,10 +25,19 @@ limitations under the License. * This base health-pill interface is general enough for all tensor * data types, including boolean, integer, float and string. */ -export interface BaseTensorHealthPill { +export interface BaseTensorNumericSummary { /** Number of elements in the tensor. */ elementCount: number; } +export interface BooleanOrNumericTensorNumericSummary + extends BaseTensorNumericSummary { + /** Minimum of all finite values. */ + minimum: number | boolean; + + /** Maximum of all finite values. */ + maximum: number | boolean; +} + // TODO(cais): Add sub-interfaces of `BaseTensorHealthPill` for other tensor // dtypes. diff --git a/tensorboard/components/tensor_widget/menu.ts b/tensorboard/components/tensor_widget/menu.ts index 26b1afe0ac..d91b5344c9 100644 --- a/tensorboard/components/tensor_widget/menu.ts +++ b/tensorboard/components/tensor_widget/menu.ts @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -type Callback = () => void | Promise; type EventCallback = (event: Event) => void | Promise; /** @@ -29,7 +28,7 @@ export interface MenuItemConfig { */ export interface SingleActionMenuItemConfig extends MenuItemConfig { /** The callback that gets called when the menu item is clicked. */ - callback: Callback; + callback: EventCallback; } export interface ChoiceMenuItemConfig extends MenuItemConfig { @@ -53,9 +52,8 @@ export interface MenuConfig { items: MenuItemConfig[]; } - interface FlatMenuItemConfig { - caption: string, + caption: string; onClick: EventCallback | null; onHover: EventCallback | null; } @@ -78,11 +76,7 @@ class FlatMenu { parentElement.appendChild(this.dropdown); } - show( - top: number, - left: number, - itemConfigs: FlatMenuItemConfig[] - ) { + show(top: number, left: number, itemConfigs: FlatMenuItemConfig[]) { itemConfigs.forEach((itemConfig, i) => { const menuItem = document.createElement('div'); menuItem.classList.add('tensor-widget-dim-dropdown-menu-item'); @@ -202,7 +196,7 @@ export class Menu { choiceConfig.callback(k); } }, - onHover: null + onHover: null, }); }); const optionsFlatMenu = new FlatMenu(parent); diff --git a/tensorboard/components/tensor_widget/tensor-widget-impl.ts b/tensorboard/components/tensor_widget/tensor-widget-impl.ts index bdaeff9ac4..7a31eb1f73 100644 --- a/tensorboard/components/tensor_widget/tensor-widget-impl.ts +++ b/tensorboard/components/tensor_widget/tensor-widget-impl.ts @@ -89,6 +89,12 @@ export class TensorWidgetImpl implements TensorWidget { // Menu configuration. protected menuConfig: MenuConfig | null = null; + // Value render mode. + protected valueRenderMode: 'text' | 'image'; + + // Size of each cell used to display the tensor value under the 'image' mode. + protected imageCellSize = 12; + constructor( private readonly rootElement: HTMLDivElement, private readonly tensorView: TensorView, @@ -97,6 +103,7 @@ export class TensorWidgetImpl implements TensorWidget { this.options = options || {}; this.slicingSpec = getDefaultSlicingSpec(this.tensorView.spec.shape); this.rank = this.tensorView.spec.shape.length; + this.valueRenderMode = 'text'; } /** @@ -110,6 +117,7 @@ export class TensorWidgetImpl implements TensorWidget { */ async render() { this.rootElement.classList.add('tensor-widget'); + this.renderHeader(); if ( !isIntegerDType(this.tensorView.spec.dtype) && @@ -143,8 +151,8 @@ export class TensorWidgetImpl implements TensorWidget { this.headerSection = document.createElement('div'); this.headerSection.classList.add('tensor-widget-header'); this.rootElement.appendChild(this.headerSection); + this.createMenu(); } - this.createMenu(); this.renderInfo(); // TODO(cais): Implement and call renderHealthPill(). } @@ -249,6 +257,12 @@ export class TensorWidgetImpl implements TensorWidget { defaultSelection: 0, callback: (currentMode: number) => { console.log(`Display mode changed: ${currentMode}`); + if (currentMode === 0) { + this.valueRenderMode = 'text'; + } else { + this.valueRenderMode = 'image'; + } + this.renderValues(); }, } as ChoiceMenuItemConfig); } @@ -475,6 +489,9 @@ export class TensorWidgetImpl implements TensorWidget { for (let i = 0; i < maxNumCols; ++i) { const tick = document.createElement('div'); tick.classList.add('tensor-widget-top-ruler-tick'); + if (this.valueRenderMode === 'image') { + tick.style.width = `${this.imageCellSize}px`; + } this.topRuler.appendChild(tick); this.topRulerTicks.push(tick); if (tick.getBoundingClientRect().right >= rootElementRight) { @@ -526,6 +543,10 @@ export class TensorWidgetImpl implements TensorWidget { for (let i = 0; i < maxNumRows; ++i) { const row = document.createElement('div'); row.classList.add('tensor-widget-value-row'); + if (this.valueRenderMode === 'image') { + row.style.height = `${this.imageCellSize}px`; + row.style.lineHeight = `${this.imageCellSize}px`; + } this.valueSection.appendChild(row); this.valueRows.push(row); @@ -577,6 +598,11 @@ export class TensorWidgetImpl implements TensorWidget { for (let j = 0; j < numCols; ++j) { const valueDiv = document.createElement('div'); valueDiv.classList.add('tensor-widget-value-div'); + if (this.valueRenderMode === 'image') { + valueDiv.style.width = `${this.imageCellSize}px`; + valueDiv.style.height = `${this.imageCellSize}px`; + valueDiv.style.lineHeight = `${this.imageCellSize}px`; + } this.valueRows[i].appendChild(valueDiv); this.valueDivs[i].push(valueDiv); valueDiv.addEventListener('click', () => { @@ -646,11 +672,13 @@ export class TensorWidgetImpl implements TensorWidget { throw new Error(`Missing horizontal range for ${this.rank}D tensor.`); } const colIndex = this.slicingSpec.horizontalRange[0] + i; - if (colIndex < numCols) { - this.topRulerTicks[i].textContent = `${colIndex}`; - } else { - this.topRulerTicks[i].textContent = ``; - } + if (this.valueRenderMode === 'text') { + if (colIndex < numCols) { + this.topRulerTicks[i].textContent = `${colIndex}`; + } else { + this.topRulerTicks[i].textContent = ``; + } + } // No text label under the image mode. } } } @@ -670,10 +698,12 @@ export class TensorWidgetImpl implements TensorWidget { throw new Error(`Missing vertcial range for ${this.rank}D tensor.`); } const rowIndex = this.slicingSpec.verticalRange[0] + i; - if (rowIndex < numRows) { - this.leftRulerTicks[i].textContent = `${rowIndex}`; - } else { - this.leftRulerTicks[i].textContent = ''; + if (this.valueRenderMode === 'text') { + if (rowIndex < numRows) { + this.leftRulerTicks[i].textContent = `${rowIndex}`; + } else { + this.leftRulerTicks[i].textContent = ''; + } } } } @@ -696,6 +726,7 @@ export class TensorWidgetImpl implements TensorWidget { } const valueClass = this.getValueClass(); + for (let i = 0; i < numRows; ++i) { for (let j = 0; j < numCols; ++j) { const valueDiv = this.valueDivs[i][j]; @@ -704,19 +735,25 @@ export class TensorWidgetImpl implements TensorWidget { j < (values as number[][])[i].length ) { const value = (values as number[][] | boolean[][] | string[][])[i][j]; - if (valueClass === 'numeric') { - // TODO(cais): Once health pills are available, use the min/max - // values to determine the number of decimal places. - valueDiv.textContent = numericValueToString( - value as number, - isIntegerDType(this.tensorView.spec.dtype) - ); - } else if (valueClass === 'boolean') { - valueDiv.textContent = booleanValueToDisplayString( - value as boolean - ); - } else if (valueClass === 'string') { - valueDiv.textContent = stringValueToDisplayString(value as string); + if (this.valueRenderMode === 'image') { + // TODO(cais): Color code. + } else { + if (valueClass === 'numeric') { + // TODO(cais): Once health pills are available, use the min/max + // values to determine the number of decimal places. + valueDiv.textContent = numericValueToString( + value as number, + isIntegerDType(this.tensorView.spec.dtype) + ); + } else if (valueClass === 'boolean') { + valueDiv.textContent = booleanValueToDisplayString( + value as boolean + ); + } else if (valueClass === 'string') { + valueDiv.textContent = stringValueToDisplayString( + value as string + ); + } } // The attribute set below will be rendered in a tooltip that appears // on mouse hovering. diff --git a/tensorboard/components/tensor_widget/tensor-widget.css b/tensorboard/components/tensor_widget/tensor-widget.css index d3354c35f3..53333d17bc 100644 --- a/tensorboard/components/tensor_widget/tensor-widget.css +++ b/tensorboard/components/tensor_widget/tensor-widget.css @@ -117,6 +117,7 @@ limitations under the License. font-weight: bold; font-size: 16px; margin-left: 10px; + margin-right: 5px; position: relative; user-select: none; } diff --git a/tensorboard/components/tensor_widget/types.ts b/tensorboard/components/tensor_widget/types.ts index 927460cceb..6760db9fec 100644 --- a/tensorboard/components/tensor_widget/types.ts +++ b/tensorboard/components/tensor_widget/types.ts @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -import {BaseTensorHealthPill} from './health-pill-types'; +import {BaseTensorNumericSummary} from './health-pill-types'; export type Shape = ReadonlyArray; @@ -58,7 +58,7 @@ export interface TensorView { view: (slicingSpec: TensorViewSlicingSpec) => Promise; /** Get the health pill of the underlying tensor. */ - getHealthPill: () => Promise; + getNumericSummary: () => Promise; } /** From a7091e13273ed067d3bf92480ecefb3bca12ea70 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 4 Oct 2019 16:02:57 -0400 Subject: [PATCH 06/16] Added basic image support --- .../components/tensor_widget/demo/demo.ts | 6 +-- .../tensor_widget/tensor-widget-impl.ts | 47 ++++++++++++++++--- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/tensorboard/components/tensor_widget/demo/demo.ts b/tensorboard/components/tensor_widget/demo/demo.ts index bd8d9ce358..7ace741af6 100644 --- a/tensorboard/components/tensor_widget/demo/demo.ts +++ b/tensorboard/components/tensor_widget/demo/demo.ts @@ -177,15 +177,15 @@ export function tensorToTensorView(x: any): tensorWidget.TensorView { }, getNumericSummary: async () => { const numericSummary: BaseTensorNumericSummary = { - elementCount: x.size(), + elementCount: x.size, }; if (x.dtype === 'float32' || x.dtype === 'int32') { (numericSummary as BooleanOrNumericTensorNumericSummary).minimum = x .min() - .dataSync(); + .arraySync(); (numericSummary as BooleanOrNumericTensorNumericSummary).maximum = x .max() - .dataSync(); + .arraySync(); } // TODO(cais): Take care of boolean dtype. return numericSummary; diff --git a/tensorboard/components/tensor_widget/tensor-widget-impl.ts b/tensorboard/components/tensor_widget/tensor-widget-impl.ts index 7a31eb1f73..e1c0b24f54 100644 --- a/tensorboard/components/tensor_widget/tensor-widget-impl.ts +++ b/tensorboard/components/tensor_widget/tensor-widget-impl.ts @@ -40,6 +40,11 @@ import { TensorWidgetOptions, TensorViewSlicingSpec, } from './types'; +import { + BaseTensorNumericSummary, + BooleanOrNumericTensorNumericSummary, +} from './health-pill-types'; +import {ColorMap, GrayscaleColorMap} from './colormap'; const DETAILED_VALUE_ATTR_KEY = 'detailed-value'; @@ -88,6 +93,8 @@ export class TensorWidgetImpl implements TensorWidget { // Menu configuration. protected menuConfig: MenuConfig | null = null; + // Menu object. + private menu: Menu | null = null; // Value render mode. protected valueRenderMode: 'text' | 'image'; @@ -95,6 +102,8 @@ export class TensorWidgetImpl implements TensorWidget { // Size of each cell used to display the tensor value under the 'image' mode. protected imageCellSize = 12; + protected numericSummary: BaseTensorNumericSummary | null = null; + constructor( private readonly rootElement: HTMLDivElement, private readonly tensorView: TensorView, @@ -259,10 +268,14 @@ export class TensorWidgetImpl implements TensorWidget { console.log(`Display mode changed: ${currentMode}`); if (currentMode === 0) { this.valueRenderMode = 'text'; + this.renderValues(); } else { this.valueRenderMode = 'image'; + this.tensorView.getNumericSummary().then((numericSummary) => { + this.numericSummary = numericSummary; + this.renderValues(); + }); } - this.renderValues(); }, } as ChoiceMenuItemConfig); } @@ -273,10 +286,6 @@ export class TensorWidgetImpl implements TensorWidget { this.renderMenuThumb(); } - private menu: Menu | null = null; - - private dropdown: HTMLDivElement | null = null; // TODO(cais): Move to top; - /** Render the thumb that when clicked, toggles the menu display state. */ private renderMenuThumb() { if (this.headerSection == null) { @@ -727,6 +736,26 @@ export class TensorWidgetImpl implements TensorWidget { const valueClass = this.getValueClass(); + let colorMap: ColorMap | null = null; + const valueRenderMode = this.valueRenderMode; + if (valueRenderMode === 'image') { + if (this.numericSummary == null) { + throw new Error( + 'Failed to render image representation of tensor due to ' + + 'missing numeric summary' + ); + } + const {minimum, maximum} = this + .numericSummary as BooleanOrNumericTensorNumericSummary; + if (minimum == null || maximum == null) { + throw new Error( + 'Failed to render image representation of tensor due to ' + + 'missing minimum or maximum values in numeric summary' + ); + } + colorMap = new GrayscaleColorMap(minimum as number, maximum as number); + } + for (let i = 0; i < numRows; ++i) { for (let j = 0; j < numCols; ++j) { const valueDiv = this.valueDivs[i][j]; @@ -735,9 +764,13 @@ export class TensorWidgetImpl implements TensorWidget { j < (values as number[][])[i].length ) { const value = (values as number[][] | boolean[][] | string[][])[i][j]; - if (this.valueRenderMode === 'image') { - // TODO(cais): Color code. + if (valueRenderMode === 'image') { + const [red, green, blue] = (colorMap as ColorMap).getRGB( + value as number + ); + valueDiv.style.backgroundColor = `rgb(${red}, ${green}, ${blue})`; } else { + // valueRenderMode: 'text' if (valueClass === 'numeric') { // TODO(cais): Once health pills are available, use the min/max // values to determine the number of decimal places. From d4349f21682891241fc0229c430adad8e4be63ec Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 4 Oct 2019 17:05:37 -0400 Subject: [PATCH 07/16] Save --- .../components/tensor_widget/demo/demo.ts | 27 ++++++++++++++----- .../tensor_widget/tensor-widget-impl.ts | 5 +++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tensorboard/components/tensor_widget/demo/demo.ts b/tensorboard/components/tensor_widget/demo/demo.ts index 7ace741af6..9ade597d56 100644 --- a/tensorboard/components/tensor_widget/demo/demo.ts +++ b/tensorboard/components/tensor_widget/demo/demo.ts @@ -179,13 +179,26 @@ export function tensorToTensorView(x: any): tensorWidget.TensorView { const numericSummary: BaseTensorNumericSummary = { elementCount: x.size, }; - if (x.dtype === 'float32' || x.dtype === 'int32') { - (numericSummary as BooleanOrNumericTensorNumericSummary).minimum = x - .min() - .arraySync(); - (numericSummary as BooleanOrNumericTensorNumericSummary).maximum = x - .max() - .arraySync(); + if (x.dtype === 'float32' || x.dtype === 'int32' || x.dtype == 'bool') { + // This is not an efficient way to compute the maximum and minimum of + // finite values in a tensor. But it is okay as this is just a demo. + const data = x.dataSync() as Float32Array; + let minimum = Infinity; + let maximum = -Infinity; + for (let i = 0; i < data.length; ++i) { + const value = data[i]; + if (!isFinite(value)) { + continue; + } + if (value < minimum) { + minimum = value; + } + if (value > maximum) { + maximum = value; + } + } + (numericSummary as BooleanOrNumericTensorNumericSummary).minimum = minimum; + (numericSummary as BooleanOrNumericTensorNumericSummary).maximum = maximum; } // TODO(cais): Take care of boolean dtype. return numericSummary; diff --git a/tensorboard/components/tensor_widget/tensor-widget-impl.ts b/tensorboard/components/tensor_widget/tensor-widget-impl.ts index e1c0b24f54..3e64e87d60 100644 --- a/tensorboard/components/tensor_widget/tensor-widget-impl.ts +++ b/tensorboard/components/tensor_widget/tensor-widget-impl.ts @@ -265,7 +265,6 @@ export class TensorWidgetImpl implements TensorWidget { options: ['Text', 'Image'], defaultSelection: 0, callback: (currentMode: number) => { - console.log(`Display mode changed: ${currentMode}`); if (currentMode === 0) { this.valueRenderMode = 'text'; this.renderValues(); @@ -561,6 +560,10 @@ export class TensorWidgetImpl implements TensorWidget { const tick = document.createElement('div'); tick.classList.add('tensor-widget-top-ruler-tick'); + if (this.valueRenderMode === 'image') { + tick.style.height = `${this.imageCellSize}px`; + tick.style.lineHeight = `${this.imageCellSize}px`; + } row.appendChild(tick); this.leftRulerTicks.push(tick); if (tick.getBoundingClientRect().bottom >= rootElementBottom) { From 228d070502ca2ef6cf22518ba53c8271c93c5898 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 4 Oct 2019 17:19:34 -0400 Subject: [PATCH 08/16] Add some doc strings --- .../components/tensor_widget/colormap.ts | 19 ++++++++++++ tensorboard/components/tensor_widget/menu.ts | 31 +++++++++++++++++-- .../tensor_widget/tensor-widget-impl.ts | 2 +- tensorboard/components/tensor_widget/types.ts | 2 +- 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/tensorboard/components/tensor_widget/colormap.ts b/tensorboard/components/tensor_widget/colormap.ts index 728054984a..b129177a79 100644 --- a/tensorboard/components/tensor_widget/colormap.ts +++ b/tensorboard/components/tensor_widget/colormap.ts @@ -17,7 +17,17 @@ limitations under the License. const MAX_RGB = 255; +/** + * Abstract base class for colormap. + * + * A colormap maps a numeric value onto an RGB color. + */ export abstract class ColorMap { + /** + * Constructor of ColorMap. + * @param min Minimum. Must be a finite value. + * @param max Maximum. Must be finite and >= `min`. + */ constructor(protected readonly min: number, protected readonly max: number) { if (!isFinite(min)) { throw new Error(`min value (${min}) is not finite`); @@ -30,9 +40,18 @@ export abstract class ColorMap { } } + /** + * Get the RGB value based on element value. + * @param value The element value to be mapped onto RGB color values. + * @returns RGB color values represented as a length-3 number array. + * The range of RGB values is 0 - 255. + */ abstract getRGB(value: number): [number, number, number]; } +/** + * A grayscale color map implementation. + */ export class GrayscaleColorMap extends ColorMap { getRGB(value: number): [number, number, number] { if (isNaN(value)) { diff --git a/tensorboard/components/tensor_widget/menu.ts b/tensorboard/components/tensor_widget/menu.ts index d91b5344c9..bafde608f7 100644 --- a/tensorboard/components/tensor_widget/menu.ts +++ b/tensorboard/components/tensor_widget/menu.ts @@ -52,19 +52,35 @@ export interface MenuConfig { items: MenuItemConfig[]; } +/** + * The configuration of an item of a FlatMenu. + */ interface FlatMenuItemConfig { + /** Caption of the FlatMenu item. */ caption: string; + + /** The click callback for the item. */ onClick: EventCallback | null; + + /** The hover callback for the item. */ onHover: EventCallback | null; } /** * Helper class: A menu item without hierarchy. + * + * A FlatMenu doesn't support hierarchy. The `Menu` class below, which supports + * menus with hierarchy, is built by composing this helper class. */ class FlatMenu { private isShown: boolean = false; private dropdown: HTMLDivElement; - // TODO(cais): Tie in the arg lengths. + + /** + * Constructor of FlatMenu. + * @param parentElement The parent element that the root UI element of this + * FlatMenu will be appended to as a child. + */ constructor(parentElement: HTMLDivElement) { this.dropdown = document.createElement('div'); this.dropdown.classList.add('tensor-widget-dim-dropdown'); @@ -76,6 +92,12 @@ class FlatMenu { parentElement.appendChild(this.dropdown); } + /** + * Show the FlatMenu item. + * @param top The top coordinate for the FlatMenu. + * @param left The left coordinate for the FlatMenu. + * @param itemConfigs Configuration of the menu items. + */ show(top: number, left: number, itemConfigs: FlatMenuItemConfig[]) { itemConfigs.forEach((itemConfig, i) => { const menuItem = document.createElement('div'); @@ -122,6 +144,7 @@ class FlatMenu { this.isShown = true; } + /** Hide this FlatMenu: Remove it from screen display. */ hide() { this.dropdown.style.display = 'none'; while (this.dropdown.firstChild) { @@ -130,6 +153,7 @@ class FlatMenu { this.isShown = false; } + /** Whether this FlatMenu is being shown currently. */ shown() { return this.isShown; } @@ -137,6 +161,8 @@ class FlatMenu { /** * A class for menu that supports configurable items and callbacks. + * + * Hierarchy is supported. */ export class Menu { private baseFlatMenu: FlatMenu; @@ -214,11 +240,12 @@ export class Menu { this.baseFlatMenu.show(top, left, outerItemConfigs); } - /** Hide the menu. */ + /** Hide the menu: Remove it from display. */ hide() { this.baseFlatMenu.hide(); } + /** Whether this Menu is being shown currently. */ shown(): boolean { return this.baseFlatMenu.shown(); } diff --git a/tensorboard/components/tensor_widget/tensor-widget-impl.ts b/tensorboard/components/tensor_widget/tensor-widget-impl.ts index 3e64e87d60..0b00ae3aea 100644 --- a/tensorboard/components/tensor_widget/tensor-widget-impl.ts +++ b/tensorboard/components/tensor_widget/tensor-widget-impl.ts @@ -773,7 +773,7 @@ export class TensorWidgetImpl implements TensorWidget { ); valueDiv.style.backgroundColor = `rgb(${red}, ${green}, ${blue})`; } else { - // valueRenderMode: 'text' + // Here, valueRenderMode is 'text'. if (valueClass === 'numeric') { // TODO(cais): Once health pills are available, use the min/max // values to determine the number of decimal places. diff --git a/tensorboard/components/tensor_widget/types.ts b/tensorboard/components/tensor_widget/types.ts index 6760db9fec..a6eb7cbe18 100644 --- a/tensorboard/components/tensor_widget/types.ts +++ b/tensorboard/components/tensor_widget/types.ts @@ -57,7 +57,7 @@ export interface TensorView { */ view: (slicingSpec: TensorViewSlicingSpec) => Promise; - /** Get the health pill of the underlying tensor. */ + /** Get the numeric summary of the underlying tensor. */ getNumericSummary: () => Promise; } From 839ccd4ac20fe828e6ea24cf169cac4826ecca07 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 4 Oct 2019 22:13:19 -0400 Subject: [PATCH 09/16] Tweak some doc strings --- tensorboard/components/tensor_widget/colormap.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tensorboard/components/tensor_widget/colormap.ts b/tensorboard/components/tensor_widget/colormap.ts index b129177a79..b6b3180d7a 100644 --- a/tensorboard/components/tensor_widget/colormap.ts +++ b/tensorboard/components/tensor_widget/colormap.ts @@ -20,7 +20,7 @@ const MAX_RGB = 255; /** * Abstract base class for colormap. * - * A colormap maps a numeric value onto an RGB color. + * A colormap maps a numeric value to an RGB color. */ export abstract class ColorMap { /** @@ -42,9 +42,9 @@ export abstract class ColorMap { /** * Get the RGB value based on element value. - * @param value The element value to be mapped onto RGB color values. - * @returns RGB color values represented as a length-3 number array. - * The range of RGB values is 0 - 255. + * @param value The element value to be mapped to RGB color value. + * @returns RGB color value represented as a length-3 number array. + * The range of RGB values is [0, 255]. */ abstract getRGB(value: number): [number, number, number]; } @@ -54,6 +54,8 @@ export abstract class ColorMap { */ export class GrayscaleColorMap extends ColorMap { getRGB(value: number): [number, number, number] { + // This color scheme for pathological values matches tfdbg v1's Health Pills + // feature. if (isNaN(value)) { // NaN. return [MAX_RGB, 0, 0]; From b589e54be8846fd243fc931402bd9ac6099dc5b7 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Fri, 4 Oct 2019 23:05:06 -0400 Subject: [PATCH 10/16] Fix empty menu; add zoom-in / zoom-out options --- .../components/tensor_widget/demo/demo.html | 2 +- tensorboard/components/tensor_widget/menu.ts | 28 ++++++++++++++++++- .../tensor_widget/tensor-widget-impl.ts | 28 +++++++++++++++++-- .../tensor_widget/tensor-widget.css | 4 +++ 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/tensorboard/components/tensor_widget/demo/demo.html b/tensorboard/components/tensor_widget/demo/demo.html index 4fe22316eb..a7c06326de 100644 --- a/tensorboard/components/tensor_widget/demo/demo.html +++ b/tensorboard/components/tensor_widget/demo/demo.html @@ -42,7 +42,7 @@

TensorWidget Demo

- +