Skip to content

Commit

Permalink
[Object Detection] [Image Explorer] Bounding Box Support with CanvasT…
Browse files Browse the repository at this point in the history
…ools on Flyout (#2116)

* bbox vott template ckpt

* canvastools image load ckpt

* loading image from scratch ckpt

* canvastools image loading support

* coordinate fixes

* regiondata call + path fix

* code cleanup

* image loading ckpt

* callback image loading support

* lint fixes

* bounding box working ckpt for fridge

* disabling internal imports for vott

* freeze rough ckpt

* file refactor lint fix

* auto lint fixes

* freeze cleanup

* lint fixes

* image dims arg for frontend bbox

* lint fixes

* lint fixes

* image dimension support

* imagedim support comments

* image dimension support

* image dimension support

* canvas module added

* image dimension working ckpt

* comment fix

* cleanup

* async image loading support

* lint fixes

* lint fixes

* reverted max line override

* code cleanups

* lint fixes

* auto lint fixes

* comment fixes
  • Loading branch information
Advitya17 authored Jun 15, 2023
1 parent f2224c8 commit 5aca92c
Showing 4 changed files with 139 additions and 3 deletions.
1 change: 1 addition & 0 deletions apps/dashboard/src/app/visionApplications.ts
Original file line number Diff line number Diff line change
@@ -72,6 +72,7 @@ export const visionApplications: IApplications = <const>{
class_names: fridgeObjectDetection.class_names,
feature_names: fridgeObjectDetection.feature_names,
features: fridgeObjectDetection.features,
imageDimensions: fridgeObjectDetection.imageDimensions,
images: fridgeObjectDetection.images,
object_detection_predicted_y:
fridgeObjectDetection.object_detection_predicted_y,
Original file line number Diff line number Diff line change
@@ -16,12 +16,19 @@ export const fridgeObjectDetection: IDataset = {
[92.000130390912],
[95.33849179841164]
],
imageDimensions: [
[499, 666],
[499, 666],
[499, 666],
[499, 666],
[499, 666]
],
images: fridgeObjectDetectionImages,
object_detection_predicted_y: [
[
[
3, 37.834022521972656, 312.5594787597656, 323.2925109863281,
420728454589844, 0.9872298836708069
420.728454589844, 0.9872298836708069
],
[
1, 314.43621826171875, 283.4221496582031, 449.1330261230469,
@@ -35,7 +42,7 @@ export const fridgeObjectDetection: IDataset = {
],
[
1, 218.88287353515625, 291.83258056640625, 431.8615417480469,
403671691894531, 0.9898059964179993
403.671691894531, 0.9898059964179993
]
],
[
Original file line number Diff line number Diff line change
@@ -284,7 +284,14 @@ export class FlyoutObjectDetection extends React.Component<
const editor = new CanvasTools.Editor(editorCallback);
const loadImage = async (): Promise<void> => {
if (this.state.item) {
// this.state.item.image is base64 encoded string
await FlyoutODUtils.loadImageFromBase64(this.state.item.image, editor);
FlyoutODUtils.drawBoundingBoxes(
this.state.item,
editorCallback,
editor,
this.props.dataset
);
}
};
loadImage();
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import { IComboBoxOption } from "@fluentui/react";
import { IComboBoxOption, getTheme } from "@fluentui/react";
import { IDataset, IVisionListItem } from "@responsible-ai/core-ui";
import { localization } from "@responsible-ai/localization";
import { CanvasTools } from "vott-ct";
import { Editor } from "vott-ct/lib/js/CanvasTools/CanvasTools.Editor";
import { Color } from "vott-ct/lib/js/CanvasTools/Core/Colors/Color";
import { RegionData } from "vott-ct/lib/js/CanvasTools/Core/RegionData";

export interface IFlyoutProps {
dataset: IDataset;
@@ -49,3 +52,121 @@ export function loadImageFromBase64(
image.src = `data:image/jpg;base64,${base64String}`;
});
}

export function drawBox(
editor: Editor,
dataset: IDataset,
scaleFactor: (
coordinate: number,
imageScale: number,
frameScale: number
) => number,
imageDim: [number, number],
objectLabel: number[],
annotation: string,
colorCode: string,
boxId: string
): void {
if (!dataset.imageDimensions) {
return;
}

const [frameWidth, frameHeight] = editor.getFrameSize;
const [imageWidth, imageHeight] = imageDim;

// Creating box region
const predBox = RegionData.BuildRectRegionData(
scaleFactor(objectLabel[1], imageWidth, frameWidth),
scaleFactor(objectLabel[2], imageHeight, frameHeight),
scaleFactor(objectLabel[3] - objectLabel[1], imageWidth, frameWidth),
scaleFactor(objectLabel[4] - objectLabel[2], imageHeight, frameHeight)
);

// Initializing bounding box tag
const predTag = new CanvasTools.Core.Tag(annotation, new Color(colorCode));
const predTagDesc = new CanvasTools.Core.TagsDescriptor([predTag]);

// Drawing bounding box with vott
editor.RM.addRectRegion(boxId, predBox, predTagDesc);
}

// Scales the box coordinate for correct relative position in the CanvasTools frame
export const scaleCoordinate = (
coordinate: number,
imageDim: number,
frameDim: number
): number => (coordinate / imageDim) * frameDim;

export function drawBoundingBoxes(
item: IVisionListItem,
editorCallback: HTMLDivElement,
editor: Editor,
dataset: IDataset
): void {
// Stops if the div container for the canvastools editor doesn't exist
if (!editorCallback) {
return;
}

const theme = getTheme();

// Ensuring object detection labels are populated
if (
!dataset.object_detection_predicted_y ||
!dataset.object_detection_true_y ||
!dataset.class_names
) {
return;
}

// Retrieving labels for the image in the Flyout
const predictedY: number[][] =
dataset.object_detection_predicted_y[item.index];
const trueY: number[][] = dataset.object_detection_true_y[item.index];

// Drawing bounding boxes for each ground truth object
for (const [oidx, gtObject] of trueY.entries()) {
if (!dataset.imageDimensions) {
break;
}
const objectLabelIndex = gtObject[0] - 1;

// Retrieving label for annotation above the box
const annotation: string = dataset.class_names[objectLabelIndex];

drawBox(
editor,
dataset,
scaleCoordinate,
dataset.imageDimensions[oidx],
gtObject,
annotation,
theme.palette.green,
oidx.toString()
);
}

// Draws bounding boxes for each predicted object
for (const [oidx, predObject] of predictedY.entries()) {
if (!dataset.imageDimensions) {
break;
}
const objectLabelIndex = predObject[0] - 1;

// Retrieving label for annotation above the box
const className: string = dataset.class_names[objectLabelIndex];
const confidenceScore: string = (predObject[5] * 100).toFixed(2);
const annotation = `${oidx}.${className}(${confidenceScore}%)`;

drawBox(
editor,
dataset,
scaleCoordinate,
dataset.imageDimensions[oidx],
predObject,
annotation,
theme.palette.magenta,
oidx.toString()
);
}
}

0 comments on commit 5aca92c

Please sign in to comment.