From 7dc7d500d038101125065ce9a48414560afd3562 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 19 Aug 2020 12:34:23 +0300 Subject: [PATCH 01/26] Draft version --- cvat-canvas/README.md | 58 ++- cvat-canvas/src/typescript/canvas.ts | 26 +- .../src/typescript/canvasController.ts | 11 + cvat-canvas/src/typescript/canvasModel.ts | 43 ++ cvat-canvas/src/typescript/canvasView.ts | 57 +++ cvat-canvas/src/typescript/crosshair.ts | 72 ++++ cvat-canvas/src/typescript/drawHandler.ts | 33 +- .../src/typescript/interactionHandler.ts | 248 +++++++++++ cvat-core/src/annotations-history.js | 6 + cvat-core/src/annotations.js | 14 + cvat-core/src/session.js | 26 ++ cvat-ui/src/actions/annotation-actions.ts | 19 + cvat-ui/src/actions/models-actions.ts | 3 +- .../controls-side-bar/controls-side-bar.tsx | 11 +- .../controls-side-bar/dextr-plugin.tsx | 5 +- .../controls-side-bar/tools-control.tsx | 385 ++++++++++++++++++ .../standard-workspace/styles.scss | 24 +- cvat-ui/src/cvat-canvas-wrapper.ts | 5 + cvat-ui/src/reducers/annotation-reducer.ts | 23 ++ cvat-ui/src/reducers/interfaces.ts | 3 + cvat-ui/src/reducers/models-reducer.ts | 6 +- cvat-ui/src/utils/dextr-utils.ts | 9 +- .../src/utils/interactors/config-provider.ts | 13 + .../src/utils/interactors/openvino.dextr.ts | 26 ++ cvat-ui/src/utils/typescript.ts | 5 + 25 files changed, 1054 insertions(+), 77 deletions(-) create mode 100644 cvat-canvas/src/typescript/crosshair.ts create mode 100644 cvat-canvas/src/typescript/interactionHandler.ts create mode 100644 cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx create mode 100644 cvat-ui/src/utils/interactors/config-provider.ts create mode 100644 cvat-ui/src/utils/interactors/openvino.dextr.ts create mode 100644 cvat-ui/src/utils/typescript.ts diff --git a/cvat-canvas/README.md b/cvat-canvas/README.md index 808aa5ac52d0..cbc7fd324fd5 100644 --- a/cvat-canvas/README.md +++ b/cvat-canvas/README.md @@ -46,6 +46,7 @@ Canvas itself handles: IDLE = 'idle', DRAG = 'drag', RESIZE = 'resize', + INTERACT = 'interact', DRAW = 'draw', EDIT = 'edit', MERGE = 'merge', @@ -70,6 +71,16 @@ Canvas itself handles: crosshair?: boolean; } + interface InteractionData { + shapeType: string; + numberOfShapes?: number; + mouse: { + right: 'positive' | 'negative' | 'cancel'; + left: 'positive' | 'negative' | 'cancel'; + center: 'positive' | 'negative' | 'cancel'; + }; + } + interface GroupData { enabled: boolean; resetGroup?: boolean; @@ -83,6 +94,12 @@ Canvas itself handles: enabled: boolean; } + interface InteractionResult { + points: number[]; + shapeType: string; + button: number; + }; + interface DrawnData { shapeType: string; points: number[]; @@ -104,6 +121,7 @@ Canvas itself handles: grid(stepX: number, stepY: number): void; draw(drawData: DrawData): void; + interact(interactionData: InteractionData): void; group(groupData: GroupData): void; split(splitData: SplitData): void; merge(mergeData: MergeData): void; @@ -146,6 +164,7 @@ Standard JS events are used. - canvas.moved => {states: ObjectState[], x: number, y: number} - canvas.find => {states: ObjectState[], x: number, y: number} - canvas.drawn => {state: DrawnData} + - canvas.interacted => {shapes: InteractionResult[]} - canvas.editstart - canvas.edited => {state: ObjectState, points: number[]} - canvas.splitted => {state: ObjectState} @@ -187,25 +206,26 @@ Standard JS events are used. ## API Reaction -| | IDLE | GROUP | SPLIT | DRAW | MERGE | EDIT | DRAG | RESIZE | ZOOM_CANVAS | DRAG_CANVAS | -|--------------|------|-------|-------|------|-------|------|------|--------|-------------|-------------| -| setup() | + | + | + | +/- | + | +/- | +/- | +/- | + | + | -| activate() | + | - | - | - | - | - | - | - | - | - | -| rotate() | + | + | + | + | + | + | + | + | + | + | -| focus() | + | + | + | + | + | + | + | + | + | + | -| fit() | + | + | + | + | + | + | + | + | + | + | -| grid() | + | + | + | + | + | + | + | + | + | + | -| draw() | + | - | - | - | - | - | - | - | - | - | -| split() | + | - | + | - | - | - | - | - | - | - | -| group() | + | + | - | - | - | - | - | - | - | - | -| merge() | + | - | - | - | + | - | - | - | - | - | -| fitCanvas() | + | + | + | + | + | + | + | + | + | + | -| dragCanvas() | + | - | - | - | - | - | + | - | - | + | -| zoomCanvas() | + | - | - | - | - | - | - | + | + | - | -| cancel() | - | + | + | + | + | + | + | + | + | + | -| configure() | + | + | + | + | + | + | + | + | + | + | -| bitmap() | + | + | + | + | + | + | + | + | + | + | -| setZLayer() | + | + | + | + | + | + | + | + | + | + | +| | IDLE | GROUP | SPLIT | DRAW | MERGE | EDIT | DRAG | RESIZE | ZOOM_CANVAS | DRAG_CANVAS | INTERACT | +|--------------|------|-------|-------|------|-------|------|------|--------|-------------|-------------|----------| +| setup() | + | + | + | +/- | + | +/- | +/- | +/- | + | + | + | +| activate() | + | - | - | - | - | - | - | - | - | - | - | +| rotate() | + | + | + | + | + | + | + | + | + | + | + | +| focus() | + | + | + | + | + | + | + | + | + | + | + | +| fit() | + | + | + | + | + | + | + | + | + | + | + | +| grid() | + | + | + | + | + | + | + | + | + | + | + | +| draw() | + | - | - | + | - | - | - | - | - | - | - | +| interact() | + | - | - | - | - | - | - | - | - | - | + | +| split() | + | - | + | - | - | - | - | - | - | - | - | +| group() | + | + | - | - | - | - | - | - | - | - | - | +| merge() | + | - | - | - | + | - | - | - | - | - | - | +| fitCanvas() | + | + | + | + | + | + | + | + | + | + | + | +| dragCanvas() | + | - | - | - | - | - | + | - | - | + | - | +| zoomCanvas() | + | - | - | - | - | - | - | + | + | - | - | +| cancel() | - | + | + | + | + | + | + | + | + | + | + | +| configure() | + | + | + | + | + | + | + | + | + | + | + | +| bitmap() | + | + | + | + | + | + | + | + | + | + | + | +| setZLayer() | + | + | + | + | + | + | + | + | + | + | + | You can call setup() during editing, dragging, and resizing only to update objects, not to change a frame. You can change frame during draw only when you do not redraw an existing object diff --git a/cvat-canvas/src/typescript/canvas.ts b/cvat-canvas/src/typescript/canvas.ts index 356997666f7d..508e55648eee 100644 --- a/cvat-canvas/src/typescript/canvas.ts +++ b/cvat-canvas/src/typescript/canvas.ts @@ -8,26 +8,17 @@ import { MergeData, SplitData, GroupData, + InteractionData, + InteractionResult, CanvasModel, CanvasModelImpl, RectDrawingMethod, CuboidDrawingMethod, Configuration, } from './canvasModel'; - -import { - Master, -} from './master'; - -import { - CanvasController, - CanvasControllerImpl, -} from './canvasController'; - -import { - CanvasView, - CanvasViewImpl, -} from './canvasView'; +import { Master } from './master'; +import { CanvasController, CanvasControllerImpl } from './canvasController'; +import { CanvasView, CanvasViewImpl } from './canvasView'; import '../scss/canvas.scss'; import pjson from '../../package.json'; @@ -43,6 +34,7 @@ interface Canvas { fit(): void; grid(stepX: number, stepY: number): void; + interact(interactionData: InteractionData): void; draw(drawData: DrawData): void; group(groupData: GroupData): void; split(splitData: SplitData): void; @@ -118,6 +110,10 @@ class CanvasImpl implements Canvas { this.model.grid(stepX, stepY); } + public interact(interactionData: InteractionData): void { + this.model.interact(interactionData); + } + public draw(drawData: DrawData): void { this.model.draw(drawData); } @@ -162,4 +158,6 @@ export { RectDrawingMethod, CuboidDrawingMethod, Mode as CanvasMode, + InteractionData, + InteractionResult, }; diff --git a/cvat-canvas/src/typescript/canvasController.ts b/cvat-canvas/src/typescript/canvasController.ts index 179f9b32e869..786836d8b0cb 100644 --- a/cvat-canvas/src/typescript/canvasController.ts +++ b/cvat-canvas/src/typescript/canvasController.ts @@ -13,6 +13,7 @@ import { SplitData, GroupData, Mode, + InteractionData, } from './canvasModel'; export interface CanvasController { @@ -21,6 +22,7 @@ export interface CanvasController { readonly focusData: FocusData; readonly activeElement: ActiveElement; readonly drawData: DrawData; + readonly interactionData: InteractionData; readonly mergeData: MergeData; readonly splitData: SplitData; readonly groupData: GroupData; @@ -30,6 +32,7 @@ export interface CanvasController { zoom(x: number, y: number, direction: number): void; draw(drawData: DrawData): void; + interact(interactionData: InteractionData): void; merge(mergeData: MergeData): void; split(splitData: SplitData): void; group(groupData: GroupData): void; @@ -84,6 +87,10 @@ export class CanvasControllerImpl implements CanvasController { this.model.draw(drawData); } + public interact(interactionData: InteractionData): void { + this.model.interact(interactionData); + } + public merge(mergeData: MergeData): void { this.model.merge(mergeData); } @@ -124,6 +131,10 @@ export class CanvasControllerImpl implements CanvasController { return this.model.drawData; } + public get interactionData(): InteractionData { + return this.model.interactionData; + } + public get mergeData(): MergeData { return this.model.mergeData; } diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index ec860e77bbe5..ae3ffe1a40ff 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -69,6 +69,20 @@ export interface DrawData { redraw?: number; } +export interface InteractionData { + enabled: boolean; + shapeType?: string; + crosshair?: boolean; + numberOfShapes?: number; + result?: 'immediate' | 'deferred'; +} + +export interface InteractionResult { + points: number[]; + shapeType: string; + button: number; +}; + export interface EditData { enabled: boolean; state: any; @@ -105,6 +119,7 @@ export enum UpdateReasons { FITTED_CANVAS = 'fitted_canvas', + INTERACT = 'interact', DRAW = 'draw', MERGE = 'merge', SPLIT = 'split', @@ -126,6 +141,7 @@ export enum Mode { MERGE = 'merge', SPLIT = 'split', GROUP = 'group', + INTERACT = 'interact', DRAG_CANVAS = 'drag_canvas', ZOOM_CANVAS = 'zoom_canvas', } @@ -139,6 +155,7 @@ export interface CanvasModel { readonly focusData: FocusData; readonly activeElement: ActiveElement; readonly drawData: DrawData; + readonly interactionData: InteractionData; readonly mergeData: MergeData; readonly splitData: SplitData; readonly groupData: GroupData; @@ -162,6 +179,7 @@ export interface CanvasModel { split(splitData: SplitData): void; merge(mergeData: MergeData): void; select(objectState: any): void; + interact(interactionData: InteractionData): void; fitCanvas(width: number, height: number): void; bitmap(enabled: boolean): void; @@ -192,6 +210,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { top: number; zLayer: number | null; drawData: DrawData; + interactionData: InteractionData; mergeData: MergeData; groupData: GroupData; splitData: SplitData; @@ -242,6 +261,9 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { enabled: false, initialState: null, }, + interactionData: { + enabled: false, + }, mergeData: { enabled: false, }, @@ -490,6 +512,23 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { this.notify(UpdateReasons.DRAW); } + public interact(interactionData: InteractionData): void { + if (![Mode.IDLE, Mode.INTERACT].includes(this.data.mode)) { + throw Error(`Canvas is busy. Action: ${this.data.mode}`); + } + + if (interactionData.enabled) { + if (this.data.interactionData.enabled) { + throw new Error('Interaction has been already started'); + } else if (!interactionData.shapeType) { + throw new Error('A shape type was not specified'); + } + } + + this.data.interactionData = interactionData; + this.notify(UpdateReasons.INTERACT); + } + public split(splitData: SplitData): void { if (![Mode.IDLE, Mode.SPLIT].includes(this.data.mode)) { throw Error(`Canvas is busy. Action: ${this.data.mode}`); @@ -647,6 +686,10 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { return { ...this.data.drawData }; } + public get interactionData(): InteractionData { + return { ...this.data.interactionData }; + } + public get mergeData(): MergeData { return { ...this.data.mergeData }; } diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts index 4eb95a8fc3e0..26244dec3f37 100644 --- a/cvat-canvas/src/typescript/canvasView.ts +++ b/cvat-canvas/src/typescript/canvasView.ts @@ -16,6 +16,7 @@ import { MergeHandler, MergeHandlerImpl } from './mergeHandler'; import { SplitHandler, SplitHandlerImpl } from './splitHandler'; import { GroupHandler, GroupHandlerImpl } from './groupHandler'; import { ZoomHandler, ZoomHandlerImpl } from './zoomHandler'; +import { InteractionHandler, InteractionHandlerImpl } from './interactionHandler'; import { AutoborderHandler, AutoborderHandlerImpl } from './autoborderHandler'; import consts from './consts'; import { @@ -42,6 +43,8 @@ import { Mode, Size, Configuration, + InteractionResult, + InteractionData, } from './canvasModel'; export interface CanvasView { @@ -72,6 +75,7 @@ export class CanvasViewImpl implements CanvasView, Listener { private groupHandler: GroupHandler; private zoomHandler: ZoomHandler; private autoborderHandler: AutoborderHandler; + private interactionHandler: InteractionHandler; private activeElement: ActiveElement; private configuration: Configuration; private serviceFlags: { @@ -127,6 +131,35 @@ export class CanvasViewImpl implements CanvasView, Listener { } } + private onStopInteraction(): void { + const event: CustomEvent = new CustomEvent('canvas.canceled', { + bubbles: false, + cancelable: true, + }); + + this.canvas.dispatchEvent(event); + this.mode = Mode.IDLE; + this.controller.interact({ + enabled: false, + }); + } + + private onInteraction(shapes: InteractionResult[] | null): void { + const { zLayer } = this.controller; + if (Array.isArray(shapes)) { + const event: CustomEvent = new CustomEvent('canvas.interacted', { + bubbles: false, + cancelable: true, + detail: { + shapes, + zOrder: zLayer || 0, + }, + }); + + this.canvas.dispatchEvent(event); + } + } + private onDrawDone(data: object | null, duration: number, continueDraw?: boolean): void { const hiddenBecauseOfDraw = Object.keys(this.serviceFlags.drawHidden) .map((_clientID): number => +_clientID); @@ -373,6 +406,8 @@ export class CanvasViewImpl implements CanvasView, Listener { this.drawHandler.transform(this.geometry); this.editHandler.transform(this.geometry); this.zoomHandler.transform(this.geometry); + this.autoborderHandler.transform(this.geometry); + this.interactionHandler.transform(this.geometry); } private transformCanvas(): void { @@ -438,7 +473,9 @@ export class CanvasViewImpl implements CanvasView, Listener { // Transform handlers this.drawHandler.transform(this.geometry); this.editHandler.transform(this.geometry); + this.zoomHandler.transform(this.geometry); this.autoborderHandler.transform(this.geometry); + this.interactionHandler.transform(this.geometry); } private resizeCanvas(): void { @@ -846,6 +883,12 @@ export class CanvasViewImpl implements CanvasView, Listener { this.adoptedContent, this.geometry, ); + this.interactionHandler = new InteractionHandlerImpl( + this.onInteraction.bind(this), + this.onStopInteraction.bind(this), + this.adoptedContent, + this.geometry, + ); // Setup event handlers this.content.addEventListener('dblclick', (e: MouseEvent): void => { @@ -1063,6 +1106,18 @@ export class CanvasViewImpl implements CanvasView, Listener { this.drawHandler.draw(data, this.geometry); } } + } else if (reason === UpdateReasons.INTERACT) { + const data: InteractionData = this.controller.interactionData; + if (data.enabled && this.mode === Mode.IDLE) { + this.canvas.style.cursor = 'crosshair'; + this.mode = Mode.INTERACT; + this.interactionHandler.interact(data); + } else { + this.canvas.style.cursor = ''; + if (this.mode !== Mode.IDLE) { + this.interactionHandler.interact(data); + } + } } else if (reason === UpdateReasons.MERGE) { const data: MergeData = this.controller.mergeData; if (data.enabled) { @@ -1101,6 +1156,8 @@ export class CanvasViewImpl implements CanvasView, Listener { } else if (reason === UpdateReasons.CANCEL) { if (this.mode === Mode.DRAW) { this.drawHandler.cancel(); + } else if (this.mode === Mode.INTERACT) { + this.interactionHandler.cancel(); } else if (this.mode === Mode.MERGE) { this.mergeHandler.cancel(); } else if (this.mode === Mode.SPLIT) { diff --git a/cvat-canvas/src/typescript/crosshair.ts b/cvat-canvas/src/typescript/crosshair.ts new file mode 100644 index 000000000000..c55698f36a18 --- /dev/null +++ b/cvat-canvas/src/typescript/crosshair.ts @@ -0,0 +1,72 @@ +// Copyright (C) 2019-2020 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import consts from './consts'; + +export default class Crosshair { + private x: SVG.Line | null; + private y: SVG.Line | null; + private canvas: SVG.Container | null; + + constructor() { + this.x = null; + this.y = null; + this.canvas = null; + } + + show(canvas: SVG.Container, x: number, y: number, scale: number) { + if (this.canvas && this.canvas !== canvas) { + if (this.x) this.x.remove(); + if (this.y) this.y.remove(); + this.x = null; + this.y = null; + } + + this.canvas = canvas; + this.x = this.canvas.line(0, y, this.canvas.node.clientWidth, y).attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * scale), + zOrder: Number.MAX_SAFE_INTEGER, + }).addClass('cvat_canvas_crosshair'); + + this.y = this.canvas.line(x, 0, x, this.canvas.node.clientHeight).attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * scale), + zOrder: Number.MAX_SAFE_INTEGER, + }).addClass('cvat_canvas_crosshair'); + } + + hide() { + if (this.x) { + this.x.remove(); + this.x = null; + } + + if (this.y) { + this.y.remove(); + this.y = null; + } + + this.canvas = null; + } + + move(x: number, y: number) { + if (this.x) { + this.x.attr({ y1: y, y2: y }); + } + + if (this.y) { + this.y.attr({ x1: x, x2: x }); + } + } + + scale(scale: number) { + if (this.x) { + this.x.attr('stroke-width', consts.BASE_STROKE_WIDTH / (2 * scale)); + } + + if (this.y) { + this.y.attr('stroke-width', consts.BASE_STROKE_WIDTH / (2 * scale)); + } + } +} diff --git a/cvat-canvas/src/typescript/drawHandler.ts b/cvat-canvas/src/typescript/drawHandler.ts index 404157bd6110..6716ead0622a 100644 --- a/cvat-canvas/src/typescript/drawHandler.ts +++ b/cvat-canvas/src/typescript/drawHandler.ts @@ -16,6 +16,7 @@ import { BBox, Box, } from './shared'; +import Crosshair from './crosshair'; import consts from './consts'; import { DrawData, @@ -44,10 +45,7 @@ export class DrawHandlerImpl implements DrawHandler { x: number; y: number; }; - private crosshair: { - x: SVG.Line; - y: SVG.Line; - }; + private crosshair: Crosshair; private drawData: DrawData; private geometry: Geometry; private autoborderHandler: AutoborderHandler; @@ -188,22 +186,11 @@ export class DrawHandlerImpl implements DrawHandler { private addCrosshair(): void { const { x, y } = this.cursorPosition; - this.crosshair = { - x: this.canvas.line(0, y, this.canvas.node.clientWidth, y).attr({ - 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * this.geometry.scale), - zOrder: Number.MAX_SAFE_INTEGER, - }).addClass('cvat_canvas_crosshair'), - y: this.canvas.line(x, 0, x, this.canvas.node.clientHeight).attr({ - 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * this.geometry.scale), - zOrder: Number.MAX_SAFE_INTEGER, - }).addClass('cvat_canvas_crosshair'), - }; + this.crosshair.show(this.canvas, x, y, this.geometry.scale); } private removeCrosshair(): void { - this.crosshair.x.remove(); - this.crosshair.y.remove(); - this.crosshair = null; + this.crosshair.hide(); } private release(): void { @@ -741,7 +728,7 @@ export class DrawHandlerImpl implements DrawHandler { this.canceled = false; this.drawData = null; this.geometry = null; - this.crosshair = null; + this.crosshair = new Crosshair(); this.drawInstance = null; this.pointsGroup = null; this.cursorPosition = { @@ -756,8 +743,7 @@ export class DrawHandlerImpl implements DrawHandler { ); this.cursorPosition = { x, y }; if (this.crosshair) { - this.crosshair.x.attr({ y1: y, y2: y }); - this.crosshair.y.attr({ x1: x, x2: x }); + this.crosshair.move(x, y); } }); } @@ -787,12 +773,7 @@ export class DrawHandlerImpl implements DrawHandler { } if (this.crosshair) { - this.crosshair.x.attr({ - 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * geometry.scale), - }); - this.crosshair.y.attr({ - 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * geometry.scale), - }); + this.crosshair.scale(this.geometry.scale); } if (this.pointsGroup) { diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts new file mode 100644 index 000000000000..b5e67579e582 --- /dev/null +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -0,0 +1,248 @@ +// Copyright (C) 2020 Intel Corporation +// +// SPDX-License-Identifier: MIT + +import * as SVG from 'svg.js'; +import consts from './consts'; +import Crosshair from './crosshair'; +import { translateToSVG } from './shared'; +import { InteractionData, InteractionResult, Geometry } from './canvasModel'; + +export interface InteractionHandler { + transform(geometry: Geometry): void; + interact(interactData: InteractionData): void; + cancel(): void; +} + +export class InteractionHandlerImpl implements InteractionHandler { + private onInteraction: (shapes: InteractionResult[] | null) => void; + onStopInteraction: () => void; + private geometry: Geometry; + private canvas: SVG.Container; + private interactionData: InteractionData; + private cursorPosition: { x: number; y: number }; + private interactionShapes: SVG.Shape[]; + private currentInteractionShape: SVG.Shape | null; + private crosshair: Crosshair; + + private prepareResult(): InteractionResult[] { + return this.interactionShapes.map((shape: SVG.Shape): InteractionResult => { + if (shape.type === 'circle') { + const points = [(shape as SVG.Circle).cx(), (shape as SVG.Circle).cy()]; + return { + points: points.map((coord: number): number => coord - this.geometry.offset), + shapeType: 'points', + button: (shape.style('stroke') as any as string) === 'red' ? 2 : 0, + }; + } else { + const bbox = (shape.node as any as SVGRectElement).getBBox(); + const points = [bbox.x, bbox.y, bbox.x + bbox.width, bbox.y + bbox.height]; + return { + points: points.map((coord: number): number => coord - this.geometry.offset), + shapeType: 'rectangle', + button: 0, + }; + } + }); + } + + private addCrosshair(): void { + const { x, y } = this.cursorPosition; + this.crosshair.show(this.canvas, x, y, this.geometry.scale); + } + + private removeCrosshair(): void { + this.crosshair.hide(); + } + + private interactPoints(): void { + const eventListener = (e: MouseEvent): void => { + if ((e.button === 0 || e.button === 2) && !e.altKey) { + const [cx, cy] = translateToSVG(this.canvas.node as any as SVGSVGElement, [e.clientX, e.clientY]); + this.currentInteractionShape = this.canvas + .circle(consts.BASE_POINT_SIZE * 2 / this.geometry.scale).center(cx, cy) + .fill('white') + .stroke(e.button === 0 ? 'green' : 'red') + .addClass('cvat_interaction_point') + .attr({ + 'stroke-width': consts.POINTS_STROKE_WIDTH / this.geometry.scale, + }); + + this.interactionShapes.push(this.currentInteractionShape); + if (this.interactionData.result === 'immediate') { + this.onInteraction(this.prepareResult()); + } + + if (typeof (this.interactionData.numberOfShapes) !== 'undefined' + && this.interactionShapes.length >= this.interactionData.numberOfShapes) { + if (this.interactionData.result !== 'immediate') { + this.onInteraction(this.prepareResult()); + } + + this.interact({ enabled: false }); + return; + } + + const self = this.currentInteractionShape; + self.on('mouseenter', (): void => { + self.attr({ + 'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / this.geometry.scale, + }); + + self.on('mousedown', (e: MouseEvent) => { + e.stopPropagation(); + self.remove(); + this.interactionShapes = this.interactionShapes.filter( + (shape: SVG.Shape): boolean => shape !== self + ); + }); + }); + + self.on('mouseleave', (): void => { + self.attr({ + 'stroke-width': consts.POINTS_STROKE_WIDTH / this.geometry.scale, + }); + + self.off('mousedown'); + }); + } + }; + + // clear this listener in relese() + this.canvas.on('mousedown.interaction', eventListener); + } + + private interactRectangle(): void { + let initialized = false; + const eventListener = (e: MouseEvent): void => { + if (e.button === 0 && !e.altKey) { + if (!initialized) { + (this.currentInteractionShape as any).draw(e, { snapToGrid: 0.1 }); + initialized = true; + } else { + (this.currentInteractionShape as any).draw(e); + } + } + }; + + this.currentInteractionShape = this.canvas.rect(); + this.canvas.on('mousedown.interaction', eventListener); + this.currentInteractionShape.on('drawstop', (): void => { + this.interactionShapes.push(this.currentInteractionShape); + + this.canvas.off('mousedown.interaction', eventListener); + if (this.interactionData.result === 'immediate') { + this.onInteraction(this.prepareResult()); + } + + if (typeof (this.interactionData.numberOfShapes) === 'undefined' + || this.interactionShapes.length < this.interactionData.numberOfShapes) { + this.interactRectangle(); + } else { + this.interact({ enabled: false }); + } + }).addClass('cvat_canvas_shape_drawing').attr({ + 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, + }); + } + + private initInteraction(): void { + if (this.interactionData.crosshair) { + this.addCrosshair(); + } + } + + private startInteraction(): void { + if (this.interactionData.shapeType === 'rectangle') { + this.interactRectangle(); + } else if (this.interactionData.shapeType === 'points') { + this.interactPoints(); + } else { + throw new Error('Interactor implementation supports only rectangle and points'); + } + } + + private release(): void { + if (this.crosshair) { + this.removeCrosshair(); + } + + this.canvas.off('mousedown.interaction'); + this.interactionShapes.forEach((shape: SVG.Shape): SVG.Shape => shape.remove()); + this.interactionShapes = []; + if (this.currentInteractionShape) { + this.currentInteractionShape.remove(); + this.currentInteractionShape = null; + } + + this.onStopInteraction(); + } + + public constructor( + onInteraction: (shapes: InteractionResult[] | null) => void, + onStopInteraction: () => void, + canvas: SVG.Container, + geometry: Geometry, + ) { + this.onInteraction = onInteraction; + this.onStopInteraction = onStopInteraction; + this.canvas = canvas; + this.geometry = geometry; + this.interactionShapes = []; + this.currentInteractionShape = null; + this.crosshair = new Crosshair(); + this.cursorPosition = { + x: 0, + y: 0, + }; + + this.canvas.on('mousemove.interaction', (e: MouseEvent): void => { + const [x, y] = translateToSVG( + this.canvas.node as any as SVGSVGElement, + [e.clientX, e.clientY], + ); + this.cursorPosition = { x, y }; + if (this.crosshair) { + this.crosshair.move(x, y); + } + }); + } + + public transform(geometry: Geometry): void { + this.geometry = geometry; + + if (this.crosshair) { + this.crosshair.scale(this.geometry.scale); + } + + const shapesToBeScaled = this.currentInteractionShape ? + [...this.interactionShapes, this.currentInteractionShape] : [...this.interactionShapes]; + for (const shape of shapesToBeScaled) { + if (shape.type === 'circle') { + (shape as SVG.Circle).radius(consts.BASE_POINT_SIZE / this.geometry.scale); + shape.attr('stroke-width', consts.POINTS_STROKE_WIDTH / this.geometry.scale); + } else { + shape.attr('stroke-width', consts.BASE_STROKE_WIDTH / this.geometry.scale); + } + } + } + + public interact(interactionData: InteractionData): void { + if (interactionData.enabled) { + this.interactionData = interactionData; + this.initInteraction(); + this.startInteraction(); + } else { + if (this.interactionData.result !== 'immediate') { + this.onInteraction(this.prepareResult()); + } + + this.release(); + this.interactionData = interactionData; + } + } + + public cancel(): void { + this.release(); + } +} diff --git a/cvat-core/src/annotations-history.js b/cvat-core/src/annotations-history.js index 4fdbf34c193f..ee84fddc36ba 100644 --- a/cvat-core/src/annotations-history.js +++ b/cvat-core/src/annotations-history.js @@ -7,9 +7,14 @@ const MAX_HISTORY_LENGTH = 128; class AnnotationHistory { constructor() { + this.frozen = false; this.clear(); } + freeze(frozen) { + this.frozen = frozen; + } + get() { return { undo: this._undo.map((undo) => [undo.action, undo.frame]), @@ -18,6 +23,7 @@ class AnnotationHistory { } do(action, undo, redo, clientIDs, frame) { + if (this.frozen) return; const actionItem = { clientIDs, action, diff --git a/cvat-core/src/annotations.js b/cvat-core/src/annotations.js index 9d7eadbf6d0d..63a316ace065 100644 --- a/cvat-core/src/annotations.js +++ b/cvat-core/src/annotations.js @@ -327,6 +327,19 @@ ); } + function freezeHistory(session, frozen) { + const sessionType = session instanceof Task ? 'task' : 'job'; + const cache = getCache(sessionType); + + if (cache.has(session)) { + return cache.get(session).history.freeze(frozen); + } + + throw new DataError( + 'Collection has not been initialized yet. Call annotations.get() or annotations.clear(true) before', + ); + } + function clearActions(session) { const sessionType = session instanceof Task ? 'task' : 'job'; const cache = getCache(sessionType); @@ -372,6 +385,7 @@ exportDataset, undoActions, redoActions, + freezeHistory, clearActions, getActions, closeSession, diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js index 6a994e0a88c6..451878965e45 100644 --- a/cvat-core/src/session.js +++ b/cvat-core/src/session.js @@ -170,6 +170,11 @@ .apiWrapper.call(this, prototype.actions.redo, count); return result; }, + async freeze(frozen) { + const result = await PluginRegistry + .apiWrapper.call(this, prototype.actions.freeze, frozen); + return result; + }, async clear() { const result = await PluginRegistry .apiWrapper.call(this, prototype.actions.clear); @@ -545,6 +550,14 @@ * @instance * @async */ + /** + * Freeze history (do not save new actions) + * @method freeze + * @memberof Session.actions + * @throws {module:API.cvat.exceptions.PluginError} + * @instance + * @async + */ /** * Remove all actions from history * @method clear @@ -745,6 +758,7 @@ this.actions = { undo: Object.getPrototypeOf(this).actions.undo.bind(this), redo: Object.getPrototypeOf(this).actions.redo.bind(this), + freeze: Object.getPrototypeOf(this).actions.freeze.bind(this), clear: Object.getPrototypeOf(this).actions.clear.bind(this), get: Object.getPrototypeOf(this).actions.get.bind(this), }; @@ -1299,6 +1313,7 @@ this.actions = { undo: Object.getPrototypeOf(this).actions.undo.bind(this), redo: Object.getPrototypeOf(this).actions.redo.bind(this), + freeze: Object.getPrototypeOf(this).actions.freeze.bind(this), clear: Object.getPrototypeOf(this).actions.clear.bind(this), get: Object.getPrototypeOf(this).actions.get.bind(this), }; @@ -1390,6 +1405,7 @@ exportDataset, undoActions, redoActions, + freezeHistory, clearActions, getActions, closeSession, @@ -1582,6 +1598,11 @@ return result; }; + Job.prototype.actions.freeze.implementation = function (frozen) { + const result = freezeHistory(this, frozen); + return result; + }; + Job.prototype.actions.clear.implementation = function () { const result = clearActions(this); return result; @@ -1846,6 +1867,11 @@ return result; }; + Task.prototype.actions.freeze.implementation = function (frozen) { + const result = freezeHistory(this, frozen); + return result; + }; + Task.prototype.actions.clear.implementation = function () { const result = clearActions(this); return result; diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 5aa6c4575023..01a9abf3d53b 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -20,6 +20,7 @@ import { Rotation, ContextMenuType, Workspace, + Model, } from 'reducers/interfaces'; import getCore from 'cvat-core-wrapper'; @@ -189,6 +190,7 @@ export enum AnnotationActionTypes { CHANGE_WORKSPACE = 'CHANGE_WORKSPACE', SAVE_LOGS_SUCCESS = 'SAVE_LOGS_SUCCESS', SAVE_LOGS_FAILED = 'SAVE_LOGS_FAILED', + INTERACT_WITH_CANVAS = 'INTERACT_WITH_CANVAS', } export function saveLogsAsync(): ThunkAction { @@ -1425,6 +1427,16 @@ export function pasteShapeAsync(): ThunkAction { }; } +export function interactWithCanvas(activeInteractor: Model, activeLabelID: number): AnyAction { + return { + type: AnnotationActionTypes.INTERACT_WITH_CANVAS, + payload: { + activeInteractor, + activeLabelID, + }, + }; +} + export function repeatDrawShapeAsync(): ThunkAction { return async (dispatch: ActionCreator): Promise => { const { @@ -1441,6 +1453,7 @@ export function repeatDrawShapeAsync(): ThunkAction { }, }, drawing: { + activeInteractor, activeObjectType, activeLabelID, activeShapeType, @@ -1450,6 +1463,12 @@ export function repeatDrawShapeAsync(): ThunkAction { } = getStore().getState().annotation; let activeControl = ActiveControl.CURSOR; + if (activeInteractor) { + canvasInstance.interact({ enabled: true, shapeType: 'points', result: 'deferred' }); + dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + return; + } + if (activeShapeType === ShapeType.RECTANGLE) { activeControl = ActiveControl.DRAW_RECTANGLE; } else if (activeShapeType === ShapeType.POINTS) { diff --git a/cvat-ui/src/actions/models-actions.ts b/cvat-ui/src/actions/models-actions.ts index 1f2afe142a45..8bc164961b49 100644 --- a/cvat-ui/src/actions/models-actions.ts +++ b/cvat-ui/src/actions/models-actions.ts @@ -84,8 +84,7 @@ export function getModelsAsync(): ThunkAction { dispatch(modelsActions.getModels()); try { - const models = (await core.lambda.list()) - .filter((model: Model) => ['detector', 'reid'].includes(model.type)); + const models = await core.lambda.list(); dispatch(modelsActions.getModelsSuccess(models)); } catch (error) { dispatch(modelsActions.getModelsFailed(error)); diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx index f6ffeb2ae5bc..a202d5c99ac4 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx @@ -14,6 +14,7 @@ import CursorControl from './cursor-control'; import MoveControl from './move-control'; import FitControl from './fit-control'; import ResizeControl from './resize-control'; +import ToolsControl from './tools-control'; import DrawRectangleControl from './draw-rectangle-control'; import DrawPolygonControl from './draw-polygon-control'; import DrawPolylineControl from './draw-polyline-control'; @@ -84,7 +85,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { preventDefault(event); const drawing = [ActiveControl.DRAW_POINTS, ActiveControl.DRAW_POLYGON, ActiveControl.DRAW_POLYLINE, ActiveControl.DRAW_RECTANGLE, - ActiveControl.DRAW_CUBOID].includes(activeControl); + ActiveControl.DRAW_CUBOID, ActiveControl.INTERACTION].includes(activeControl); if (!drawing) { canvasInstance.cancel(); @@ -97,6 +98,12 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { repeatDrawShape(); } } else { + if (activeControl === ActiveControl.INTERACTION) { + // separated API method + canvasInstance.interact({ enabled: false }); + return; + } + canvasInstance.draw({ enabled: false }); } }, @@ -178,7 +185,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
- + ({ + model, + config: configProvider(model.id), + })), + isInteraction: activeControl === ActiveControl.INTERACTION, + activeLabelID: annotation.drawing.activeLabelID, + labels: annotation.job.labels, + states: annotation.annotations.states, + canvasInstance, + jobInstance, + frame, + }; +} + +function mapDispatchToProps(dispatch: any): DispatchToProps { + return { + onInteractionStart(activeInteractor: Model, activeLabelID: number): void { + dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + }, + updateAnnotations(statesToUpdate: any[]): void { + dispatch(updateAnnotationsAsync(statesToUpdate)); + }, + fetchAnnotations(): void { + dispatch(fetchAnnotationsAsync()); + }, + }; +} + +function convertShapesForInteractor(shapes: InteractionResult[]): number[][] { + const reducer = (acc: number[][], _: number, index: number, array: number[]): number[][] => { + if (!(index % 2)) { // 0, 2, 4 + acc.push([ + array[index], + array[index + 1], + ]); + } + return acc; + }; + + return shapes.filter((shape: InteractionResult): boolean => shape.shapeType === 'points') + .map((shape: InteractionResult): number[] => shape.points) + .flat().reduce(reducer, []); +} + +type Props = StateToProps & DispatchToProps; +interface State { + activeInteractor: Interactor | null; + activeLabelID: number; + interactiveStateID: number | null; + fetching: boolean; +} + +class ToolsControlComponent extends React.PureComponent { + public constructor(props: Props) { + super(props); + this.state = { + activeInteractor: props.interactors.length ? props.interactors[0] : null, + activeLabelID: props.labels[0].id, + interactiveStateID: null, + fetching: false, + }; + } + + public componentDidMount(): void { + const { canvasInstance } = this.props; + canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener); + } + + public componentDidUpdate(prevProps: Props): void { + const icon = window.document.getElementsByClassName('cvat-tools-control')[0]; + if (icon) { + const svg: SVGSVGElement = icon.children[0] as SVGSVGElement; + svg.setAttribute('width', '40px'); + svg.setAttribute('height', '40px'); + } + + const { isInteraction, jobInstance } = this.props; + const { interactiveStateID } = this.state; + if (prevProps.isInteraction && !isInteraction) { + if (interactiveStateID !== null) { + jobInstance.actions.freeze(false); + this.setState({ + interactiveStateID: null, + }); + } + } + } + + public componentWillUnmount(): void { + const { canvasInstance } = this.props; + canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener); + } + + private interactionListener = async (e: Event): Promise => { + const { + frame, + states, + labels, + jobInstance, + isInteraction, + activeLabelID, + fetchAnnotations, + updateAnnotations, + } = this.props; + const { activeInteractor, interactiveStateID } = this.state; + + try { + this.setState({ + fetching: true, + }); + + if (!isInteraction) { + throw Error('Canvas raises "canvas.interacted" when interaction is off'); + } + + const interactor = activeInteractor as Interactor; + const result = await core.lambda.call(jobInstance.task, interactor.model, { + task: jobInstance.task, + frame, + points: convertShapesForInteractor((e as CustomEvent).detail.shapes), + }); + + if (interactiveStateID === null) { + const object = new core.classes.ObjectState({ + frame, + objectType: ObjectType.SHAPE, + label: labels + .filter((label: any) => label.id === activeLabelID)[0], + shapeType: ShapeType.POLYGON, + points: result.flat(), + occluded: false, + zOrder: (e as CustomEvent).detail.zOrder, + }); + const [clientID] = await jobInstance.annotations.put([object]); + fetchAnnotations(); + await jobInstance.actions.freeze(true); // only if still interaction + this.setState({ + interactiveStateID: clientID, + }); + } else { + const state = states + .filter((_state: any): boolean => _state.clientID === interactiveStateID)[0]; + + state.points = result.flat(); + await updateAnnotations([state]); + // TODO: disable switching between frames + } + } catch (err) { + notification.error({ + description: err.toString(), + message: 'Interaction error occured', + }); + } finally { + this.setState({ + fetching: false, + }); + } + }; + + private setActiveInteractor = (key: string): void => { + const { interactors } = this.props; + this.setState({ + activeInteractor: interactors.filter( + (interactor: Interactor) => interactor.model.id === key, + )[0], + }); + }; + + private setActiveLabel = (key: string): void => { + const { labels } = this.props; + this.setState({ + activeLabelID: labels.filter( + (label: any) => label.id === +key, + )[0], + }); + }; + + private renderLabelBlock(): JSX.Element { + const { labels, activeLabelID } = this.props; + return ( + <> + + + Label + + + + + + + + + ); + } + + private renderInteractorBlock(): JSX.Element { + const { interactors, canvasInstance, onInteractionStart } = this.props; + const { activeInteractor, activeLabelID, fetching } = this.state; + + return ( + <> + + + Interactor + + + + + + + + + + + + + + ); + } + + private renderPopoverContent(): JSX.Element { + return ( +
+ + + AI Tools + + + { this.renderLabelBlock() } + { this.renderInteractorBlock() } +
+ ); + } + + public render(): JSX.Element | null { + const { interactors, isInteraction, canvasInstance } = this.props; + if (!interactors.length) return null; + + const dynamcPopoverPros = isInteraction ? { + overlayStyle: { + display: 'none', + }, + } : {}; + + const dynamicIconProps = isInteraction ? { + className: 'cvat-active-canvas-control cvat-tools-control', + onClick: (): void => { + canvasInstance.interact({ enabled: false }); + }, + } : { + className: 'cvat-tools-control', + }; + + return ( + + + + ); + } +} + +export default connect( + mapStateToProps, + mapDispatchToProps, +)(ToolsControlComponent); diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss b/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss index a9514add5d2a..71eb7e490838 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss +++ b/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss @@ -83,17 +83,31 @@ padding: 0; } -.cvat-draw-shape-popover > -.ant-popover-content > -.ant-popover-inner > div > -.ant-popover-inner-content { - padding: 0; +.cvat-draw-shape-popover, +.cvat-tools-control-popover { + > .ant-popover-content > + .ant-popover-inner > div > + .ant-popover-inner-content { + padding: 0; + } +} + +.cvat-tools-interact-button { + width: 100%; + margin-top: 10px; } .cvat-draw-shape-popover-points-selector { width: 100%; } +.cvat-tools-control-popover-content { + padding: 10px; + border-radius: 5px; + background: $background-color-2; + width: 270px; +} + .cvat-draw-shape-popover-content { padding: 10px; border-radius: 5px; diff --git a/cvat-ui/src/cvat-canvas-wrapper.ts b/cvat-ui/src/cvat-canvas-wrapper.ts index 631a52a915f3..2dc70b8db859 100644 --- a/cvat-ui/src/cvat-canvas-wrapper.ts +++ b/cvat-ui/src/cvat-canvas-wrapper.ts @@ -8,8 +8,13 @@ import { CanvasVersion, RectDrawingMethod, CuboidDrawingMethod, + InteractionData as InteractionDataType, + InteractionResult as InteractionResultType, } from 'cvat-canvas/src/typescript/canvas'; +export type InteractionData = InteractionDataType; +export type InteractionResult = InteractionResultType; + export { Canvas, CanvasMode, diff --git a/cvat-ui/src/reducers/annotation-reducer.ts b/cvat-ui/src/reducers/annotation-reducer.ts index 91756d9b6402..18e32c8cabf2 100644 --- a/cvat-ui/src/reducers/annotation-reducer.ts +++ b/cvat-ui/src/reducers/annotation-reducer.ts @@ -428,6 +428,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { activeControl, }, drawing: { + activeInteractor: undefined, activeLabelID: labelID, activeNumOfPoints: points, activeObjectType: objectType, @@ -1064,8 +1065,30 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { }, }; } + case AnnotationActionTypes.INTERACT_WITH_CANVAS: { + return { + ...state, + annotations: { + ...state.annotations, + activatedStateID: null, + }, + drawing: { + ...state.drawing, + activeInteractor: action.payload.activeInteractor, + activeLabelID: action.payload.activeLabelID, + }, + canvas: { + ...state.canvas, + activeControl: ActiveControl.INTERACTION, + }, + }; + } case AnnotationActionTypes.CHANGE_WORKSPACE: { const { workspace } = action.payload; + if (state.canvas.activeControl !== ActiveControl.CURSOR) { + return state; + } + return { ...state, workspace, diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index 72feee69a65c..a172ddd95083 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -161,6 +161,7 @@ export interface ModelsState { fetching: boolean; creatingStatus: string; models: Model[]; + interactors: Model[]; inferences: { [index: number]: ActiveInference; }; @@ -268,6 +269,7 @@ export enum ActiveControl { GROUP = 'group', SPLIT = 'split', EDIT = 'edit', + INTERACTION = 'interaction', } export enum ShapeType { @@ -340,6 +342,7 @@ export interface AnnotationState { frameAngles: number[]; }; drawing: { + activeInteractor?: Model; activeShapeType: ShapeType; activeRectDrawingMethod?: RectDrawingMethod; activeNumOfPoints?: number; diff --git a/cvat-ui/src/reducers/models-reducer.ts b/cvat-ui/src/reducers/models-reducer.ts index 3e369584b5fe..26a9775340be 100644 --- a/cvat-ui/src/reducers/models-reducer.ts +++ b/cvat-ui/src/reducers/models-reducer.ts @@ -5,13 +5,14 @@ import { boundariesActions, BoundariesActionTypes } from 'actions/boundaries-actions'; import { ModelsActionTypes, ModelsActions } from 'actions/models-actions'; import { AuthActionTypes, AuthActions } from 'actions/auth-actions'; -import { ModelsState } from './interfaces'; +import { ModelsState, Model } from './interfaces'; const defaultState: ModelsState = { initialized: false, fetching: false, creatingStatus: '', models: [], + interactors: [], visibleRunWindows: false, activeRunTask: null, inferences: {}, @@ -32,7 +33,8 @@ export default function ( case ModelsActionTypes.GET_MODELS_SUCCESS: { return { ...state, - models: action.payload.models, + models: action.payload.models.filter((model: Model) => ['detector', 'reid'].includes(model.type)), + interactors: action.payload.models.filter((model: Model) => ['interactor'].includes(model.type)), initialized: true, fetching: false, }; diff --git a/cvat-ui/src/utils/dextr-utils.ts b/cvat-ui/src/utils/dextr-utils.ts index a9d92abfffd2..eac66d7cdafa 100644 --- a/cvat-ui/src/utils/dextr-utils.ts +++ b/cvat-ui/src/utils/dextr-utils.ts @@ -31,11 +31,6 @@ interface DEXTRPlugin { }; } -interface Point { - x: number; - y: number; -} - const antModalRoot = document.createElement('div'); const antModalMask = document.createElement('div'); antModalMask.classList.add('ant-modal-mask'); @@ -209,6 +204,10 @@ export function deactivate(canvasInstance: Canvas): void { } } +export function isActivated(): boolean { + return plugin.data.enabled; +} + export function registerDEXTRPlugin(): void { core.plugins.register(plugin); } diff --git a/cvat-ui/src/utils/interactors/config-provider.ts b/cvat-ui/src/utils/interactors/config-provider.ts new file mode 100644 index 000000000000..9ab487f7a802 --- /dev/null +++ b/cvat-ui/src/utils/interactors/config-provider.ts @@ -0,0 +1,13 @@ +import dextrConfig from './openvino.dextr'; + +const configMap = { + 'openvino.dextr': dextrConfig, +}; + +export default function receiveConfig(modelId: string): object { + if (modelId in configMap) { + return (configMap as any)[modelId]; + } + + throw new Error(`Unknown model ID. Please put config for "${modelId}" and rebuild the client`); +} diff --git a/cvat-ui/src/utils/interactors/openvino.dextr.ts b/cvat-ui/src/utils/interactors/openvino.dextr.ts new file mode 100644 index 000000000000..0d3a0b4fcfd9 --- /dev/null +++ b/cvat-ui/src/utils/interactors/openvino.dextr.ts @@ -0,0 +1,26 @@ +import { InteractionResult } from 'cvat-canvas-wrapper'; + +const reducer = (acc: number[][], _: number, index: number, array: number[]): number[][] => { + if (!(index % 2)) { // 0, 2, 4 + acc.push([ + array[index], + array[index + 1], + ]); + } + return acc; +}; + +export default { + shapeType: 'points', // also 'rectangle' is avaliable + crosshair: true, + result: 'immediate', // also 'immediate' is avaliable + numberOfShapes: undefined, // can restrict number of points/rectangles + convertToSever: (shapes: InteractionResult[]) => ( + // converts output of cvat-canvas to server input + { + points: shapes.filter((shape: InteractionResult): boolean => shape.shapeType === 'points') + .map((shape: InteractionResult): number[] => shape.points) + .flat().reduce(reducer, []), + } + ), +}; diff --git a/cvat-ui/src/utils/typescript.ts b/cvat-ui/src/utils/typescript.ts new file mode 100644 index 000000000000..626135b0ccdf --- /dev/null +++ b/cvat-ui/src/utils/typescript.ts @@ -0,0 +1,5 @@ +export type Pick2 = { + [P1 in K1]: { + [P2 in K2]: (T[K1])[P2]; + }; +}; From cc78167c74f8c3c31b2c6b1de041b555cbd731d3 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 19 Aug 2020 12:46:32 +0300 Subject: [PATCH 02/26] Removed extra file --- cvat-ui/src/utils/typescript.ts | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 cvat-ui/src/utils/typescript.ts diff --git a/cvat-ui/src/utils/typescript.ts b/cvat-ui/src/utils/typescript.ts deleted file mode 100644 index 626135b0ccdf..000000000000 --- a/cvat-ui/src/utils/typescript.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type Pick2 = { - [P1 in K1]: { - [P2 in K2]: (T[K1])[P2]; - }; -}; From 2cab41f5a92a7556fbeef3d79c44084177840936 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 19 Aug 2020 17:34:22 +0300 Subject: [PATCH 03/26] Removed extra code --- cvat-canvas/src/typescript/canvasModel.ts | 1 + cvat-ui/src/actions/annotation-actions.ts | 6 +- .../controls-side-bar/dextr-plugin.tsx | 91 -------- .../draw-polygon-control.tsx | 1 + .../controls-side-bar/draw-shape-popover.tsx | 2 - .../controls-side-bar/tools-control.tsx | 34 ++- cvat-ui/src/reducers/interfaces.ts | 1 - cvat-ui/src/reducers/plugins-reducer.ts | 6 - cvat-ui/src/utils/dextr-utils.ts | 213 ------------------ .../src/utils/interactors/config-provider.ts | 13 -- .../src/utils/interactors/openvino.dextr.ts | 26 --- cvat-ui/src/utils/plugin-checker.ts | 8 - 12 files changed, 18 insertions(+), 384 deletions(-) delete mode 100644 cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/dextr-plugin.tsx delete mode 100644 cvat-ui/src/utils/dextr-utils.ts delete mode 100644 cvat-ui/src/utils/interactors/config-provider.ts delete mode 100644 cvat-ui/src/utils/interactors/openvino.dextr.ts diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index ae3ffe1a40ff..80749fc1e87a 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -526,6 +526,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { } this.data.interactionData = interactionData; + this.data.interactionData.crosshair = true; this.notify(UpdateReasons.INTERACT); } diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 01a9abf3d53b..430566de5ff2 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -1464,7 +1464,7 @@ export function repeatDrawShapeAsync(): ThunkAction { let activeControl = ActiveControl.CURSOR; if (activeInteractor) { - canvasInstance.interact({ enabled: true, shapeType: 'points', result: 'deferred' }); + canvasInstance.interact({ enabled: true }); dispatch(interactWithCanvas(activeInteractor, activeLabelID)); return; } @@ -1502,7 +1502,7 @@ export function repeatDrawShapeAsync(): ThunkAction { rectDrawingMethod: activeRectDrawingMethod, numberOfPoints: activeNumOfPoints, shapeType: activeShapeType, - crosshair: activeShapeType === ShapeType.RECTANGLE, + crosshair: [ShapeType.RECTANGLE, ShapeType.CUBOID].includes(activeShapeType), }); } }; @@ -1549,7 +1549,7 @@ export function redrawShapeAsync(): ThunkAction { enabled: true, redraw: activatedStateID, shapeType: state.shapeType, - crosshair: state.shapeType === ShapeType.RECTANGLE, + crosshair: [ShapeType.RECTANGLE, ShapeType.CUBOID].includes(state.shapeType), }); } } diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/dextr-plugin.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/dextr-plugin.tsx deleted file mode 100644 index 9c7e93747226..000000000000 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/dextr-plugin.tsx +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React, { useState } from 'react'; -import { connect } from 'react-redux'; -import Checkbox, { CheckboxChangeEvent } from 'antd/lib/checkbox'; -import Tooltip from 'antd/lib/tooltip'; - -import { Canvas } from 'cvat-canvas-wrapper'; -import { CombinedState } from 'reducers/interfaces'; -import { activate as activatePlugin, deactivate as deactivatePlugin, isActivated } from 'utils/dextr-utils'; - - -interface StateToProps { - pluginEnabled: boolean; - canvasInstance: Canvas; -} - -interface DispatchToProps { - activate(canvasInstance: Canvas): void; - deactivate(canvasInstance: Canvas): void; -} - -function mapStateToProps(state: CombinedState): StateToProps { - const { - plugins: { - list, - }, - annotation: { - canvas: { - instance: canvasInstance, - }, - }, - } = state; - - return { - canvasInstance, - pluginEnabled: list.DEXTR_SEGMENTATION, - }; -} - -function mapDispatchToProps(): DispatchToProps { - return { - activate(canvasInstance: Canvas): void { - activatePlugin(canvasInstance); - }, - deactivate(canvasInstance: Canvas): void { - deactivatePlugin(canvasInstance); - }, - }; -} - -function DEXTRPlugin(props: StateToProps & DispatchToProps): JSX.Element | null { - const { - pluginEnabled, - canvasInstance, - activate, - deactivate, - } = props; - - const [pluginActivated, setActivated] = useState(isActivated()); - - return ( - pluginEnabled ? ( - - { - setActivated(event.target.checked); - if (event.target.checked) { - activate(canvasInstance); - } else { - deactivate(canvasInstance); - } - }} - > - Make AI polygon - - - ) : null - ); -} - -export default connect( - mapStateToProps, - mapDispatchToProps, -)(DEXTRPlugin); - -// TODO: Add dialog window with cancel button diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-polygon-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-polygon-control.tsx index 1be11c2f342c..3b80a54cf31e 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-polygon-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-polygon-control.tsx @@ -43,6 +43,7 @@ function DrawPolygonControl(props: Props): JSX.Element { )} > diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx index 5edfc758735e..d8ce4d33867e 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/draw-shape-popover.tsx @@ -14,7 +14,6 @@ import Text from 'antd/lib/typography/Text'; import { RectDrawingMethod, CuboidDrawingMethod } from 'cvat-canvas-wrapper'; import { ShapeType } from 'reducers/interfaces'; import { clamp } from 'utils/math'; -import DEXTRPlugin from './dextr-plugin'; interface Props { shapeType: ShapeType; @@ -91,7 +90,6 @@ function DrawShapePopoverComponent(props: Props): JSX.Element { - { shapeType === ShapeType.POLYGON && } { shapeType === ShapeType.RECTANGLE && ( <> diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 3f2b4907ecd2..c31203f7f09a 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -14,7 +14,6 @@ import notification from 'antd/lib/notification'; import { Canvas } from 'cvat-canvas-wrapper'; import getCore from 'cvat-core-wrapper'; -import configProvider from 'utils/interactors/config-provider'; import { CombinedState, ActiveControl, @@ -25,11 +24,6 @@ import { import { interactWithCanvas, fetchAnnotationsAsync, updateAnnotationsAsync } from 'actions/annotation-actions'; import { InteractionResult } from 'cvat-canvas/src/typescript/canvas'; -interface Interactor { - model: Model; - config: object; -} - interface StateToProps { canvasInstance: Canvas; labels: any[]; @@ -38,7 +32,7 @@ interface StateToProps { jobInstance: any; isInteraction: boolean; frame: number; - interactors: Interactor[]; + interactors: Model[]; } interface DispatchToProps { @@ -58,10 +52,7 @@ function mapStateToProps(state: CombinedState): StateToProps { const { interactors } = models; return { - interactors: interactors.map((model: Model): Interactor => ({ - model, - config: configProvider(model.id), - })), + interactors, isInteraction: activeControl === ActiveControl.INTERACTION, activeLabelID: annotation.drawing.activeLabelID, labels: annotation.job.labels, @@ -104,7 +95,7 @@ function convertShapesForInteractor(shapes: InteractionResult[]): number[][] { type Props = StateToProps & DispatchToProps; interface State { - activeInteractor: Interactor | null; + activeInteractor: Model | null; activeLabelID: number; interactiveStateID: number | null; fetching: boolean; @@ -173,8 +164,8 @@ class ToolsControlComponent extends React.PureComponent { throw Error('Canvas raises "canvas.interacted" when interaction is off'); } - const interactor = activeInteractor as Interactor; - const result = await core.lambda.call(jobInstance.task, interactor.model, { + const interactor = activeInteractor as Model; + const result = await core.lambda.call(jobInstance.task, interactor, { task: jobInstance.task, frame, points: convertShapesForInteractor((e as CustomEvent).detail.shapes), @@ -221,7 +212,7 @@ class ToolsControlComponent extends React.PureComponent { const { interactors } = this.props; this.setState({ activeInteractor: interactors.filter( - (interactor: Interactor) => interactor.model.id === key, + (interactor: Model) => interactor.id === key, )[0], }); }; @@ -294,14 +285,14 @@ class ToolsControlComponent extends React.PureComponent { @@ -317,11 +308,12 @@ class ToolsControlComponent extends React.PureComponent { if (activeInteractor) { canvasInstance.cancel(); canvasInstance.interact({ - ...activeInteractor.config, + shapeType: 'points', + numberOfShapes: 4, // TODO: Add parameter to server enabled: true, }); - onInteractionStart(activeInteractor.model, activeLabelID); + onInteractionStart(activeInteractor, activeLabelID); } }} > diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index a172ddd95083..ec8138189a1e 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -75,7 +75,6 @@ export interface FormatsState { // eslint-disable-next-line import/prefer-default-export export enum SupportedPlugins { GIT_INTEGRATION = 'GIT_INTEGRATION', - DEXTR_SEGMENTATION = 'DEXTR_SEGMENTATION', ANALYTICS = 'ANALYTICS', } diff --git a/cvat-ui/src/reducers/plugins-reducer.ts b/cvat-ui/src/reducers/plugins-reducer.ts index 71cfc3aba2a9..b18db9871c69 100644 --- a/cvat-ui/src/reducers/plugins-reducer.ts +++ b/cvat-ui/src/reducers/plugins-reducer.ts @@ -4,7 +4,6 @@ import { PluginsActionTypes, PluginActions } from 'actions/plugins-actions'; import { registerGitPlugin } from 'utils/git-utils'; -import { registerDEXTRPlugin } from 'utils/dextr-utils'; import { PluginsState } from './interfaces'; const defaultState: PluginsState = { @@ -12,7 +11,6 @@ const defaultState: PluginsState = { initialized: false, list: { GIT_INTEGRATION: false, - DEXTR_SEGMENTATION: false, ANALYTICS: false, }, }; @@ -36,10 +34,6 @@ export default function ( registerGitPlugin(); } - if (!state.list.DEXTR_SEGMENTATION && list.DEXTR_SEGMENTATION) { - registerDEXTRPlugin(); - } - return { ...state, initialized: true, diff --git a/cvat-ui/src/utils/dextr-utils.ts b/cvat-ui/src/utils/dextr-utils.ts deleted file mode 100644 index eac66d7cdafa..000000000000 --- a/cvat-ui/src/utils/dextr-utils.ts +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import getCore from 'cvat-core-wrapper'; -import { Canvas } from 'cvat-canvas-wrapper'; -import { ShapeType, CombinedState } from 'reducers/interfaces'; -import { getCVATStore } from 'cvat-store'; - -const core = getCore(); - -interface DEXTRPlugin { - name: string; - description: string; - cvat: { - classes: { - Job: { - prototype: { - annotations: { - put: { - enter(self: any, objects: any[]): Promise; - }; - }; - }; - }; - }; - }; - data: { - canceled: boolean; - enabled: boolean; - }; -} - -const antModalRoot = document.createElement('div'); -const antModalMask = document.createElement('div'); -antModalMask.classList.add('ant-modal-mask'); -const antModalWrap = document.createElement('div'); -antModalWrap.classList.add('ant-modal-wrap'); -antModalWrap.setAttribute('role', 'dialog'); -const antModal = document.createElement('div'); -antModal.classList.add('ant-modal'); -antModal.style.width = '300px'; -antModal.style.top = '40%'; -antModal.setAttribute('role', 'document'); -const antModalContent = document.createElement('div'); -antModalContent.classList.add('ant-modal-content'); -const antModalBody = document.createElement('div'); -antModalBody.classList.add('ant-modal-body'); -antModalBody.style.textAlign = 'center'; -const antModalSpan = document.createElement('span'); -antModalSpan.innerText = 'Segmentation request is being processed'; -antModalSpan.style.display = 'block'; -const antModalButton = document.createElement('button'); -antModalButton.disabled = true; -antModalButton.classList.add('ant-btn', 'ant-btn-primary'); -antModalButton.style.width = '100px'; -antModalButton.style.margin = '10px auto'; -const antModalButtonSpan = document.createElement('span'); -antModalButtonSpan.innerText = 'Cancel'; - -antModalBody.append(antModalSpan, antModalButton); -antModalButton.append(antModalButtonSpan); -antModalContent.append(antModalBody); -antModal.append(antModalContent); -antModalWrap.append(antModal); -antModalRoot.append(antModalMask, antModalWrap); - -async function serverRequest( - taskInstance: any, - frame: number, - points: number[], -): Promise { - const reducer = (acc: number[][], - _: number, index: number, - array: number[]): number[][] => { - if (!(index % 2)) { // 0, 2, 4 - acc.push([ - array[index], - array[index + 1], - ]); - } - return acc; - }; - - const reducedPoints = points.reduce(reducer, []); - const models = await core.lambda.list(); - const model = models.filter((func: any): boolean => func.id === 'openvino.dextr')[0]; - const result = await core.lambda.call(taskInstance, model, { - task: taskInstance, - frame, - points: reducedPoints, - }); - - return result.flat(); -} - -async function enter(this: any, self: DEXTRPlugin, objects: any[]): Promise { - try { - if (self.data.enabled && objects.length === 1) { - const state = (getCVATStore().getState() as CombinedState); - const isPolygon = state.annotation - .drawing.activeShapeType === ShapeType.POLYGON; - if (!isPolygon) return; - - document.body.append(antModalRoot); - const promises: Record> = {}; - for (let i = 0; i < objects.length; i++) { - if (objects[i].points.length >= 8) { - promises[i] = serverRequest( - this.task, - objects[i].frame, - objects[i].points, - ); - } else { - promises[i] = new Promise((resolve) => { - resolve(objects[i].points); - }); - } - } - - const transformed = await Promise - .all(Object.values(promises)); - for (let i = 0; i < objects.length; i++) { - // eslint-disable-next-line no-param-reassign - objects[i] = new core.classes.ObjectState({ - frame: objects[i].frame, - objectType: objects[i].objectType, - label: objects[i].label, - shapeType: ShapeType.POLYGON, - points: transformed[i], - occluded: objects[i].occluded, - zOrder: objects[i].zOrder, - }); - } - } - - return; - } catch (error) { - throw new core.exceptions.PluginError(error.toString()); - } finally { - // eslint-disable-next-line no-param-reassign - self.data.canceled = false; - antModalButton.disabled = true; - if (antModalRoot.parentElement === document.body) { - document.body.removeChild(antModalRoot); - } - } -} - -const plugin: DEXTRPlugin = { - name: 'Deep extreme cut', - description: 'Plugin allows to get a polygon from extreme points using AI', - cvat: { - classes: { - Job: { - prototype: { - annotations: { - put: { - enter, - }, - }, - }, - }, - }, - }, - data: { - canceled: false, - enabled: false, - }, -}; - - -antModalButton.onclick = () => { - plugin.data.canceled = true; -}; - -export function activate(canvasInstance: Canvas): void { - if (!plugin.data.enabled) { - // eslint-disable-next-line no-param-reassign - canvasInstance.draw = (drawData: any): void => { - if (drawData.enabled && drawData.shapeType === ShapeType.POLYGON - && (typeof (drawData.numberOfPoints) === 'undefined' || drawData.numberOfPoints >= 4) - && (typeof (drawData.initialState) === 'undefined') - ) { - const patchedData = { ...drawData }; - patchedData.shapeType = ShapeType.POINTS; - patchedData.crosshair = true; - Object.getPrototypeOf(canvasInstance) - .draw.call(canvasInstance, patchedData); - } else { - Object.getPrototypeOf(canvasInstance) - .draw.call(canvasInstance, drawData); - } - }; - plugin.data.enabled = true; - } -} - -export function deactivate(canvasInstance: Canvas): void { - if (plugin.data.enabled) { - // eslint-disable-next-line no-param-reassign - canvasInstance.draw = Object.getPrototypeOf(canvasInstance).draw; - plugin.data.enabled = false; - } -} - -export function isActivated(): boolean { - return plugin.data.enabled; -} - -export function registerDEXTRPlugin(): void { - core.plugins.register(plugin); -} diff --git a/cvat-ui/src/utils/interactors/config-provider.ts b/cvat-ui/src/utils/interactors/config-provider.ts deleted file mode 100644 index 9ab487f7a802..000000000000 --- a/cvat-ui/src/utils/interactors/config-provider.ts +++ /dev/null @@ -1,13 +0,0 @@ -import dextrConfig from './openvino.dextr'; - -const configMap = { - 'openvino.dextr': dextrConfig, -}; - -export default function receiveConfig(modelId: string): object { - if (modelId in configMap) { - return (configMap as any)[modelId]; - } - - throw new Error(`Unknown model ID. Please put config for "${modelId}" and rebuild the client`); -} diff --git a/cvat-ui/src/utils/interactors/openvino.dextr.ts b/cvat-ui/src/utils/interactors/openvino.dextr.ts deleted file mode 100644 index 0d3a0b4fcfd9..000000000000 --- a/cvat-ui/src/utils/interactors/openvino.dextr.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { InteractionResult } from 'cvat-canvas-wrapper'; - -const reducer = (acc: number[][], _: number, index: number, array: number[]): number[][] => { - if (!(index % 2)) { // 0, 2, 4 - acc.push([ - array[index], - array[index + 1], - ]); - } - return acc; -}; - -export default { - shapeType: 'points', // also 'rectangle' is avaliable - crosshair: true, - result: 'immediate', // also 'immediate' is avaliable - numberOfShapes: undefined, // can restrict number of points/rectangles - convertToSever: (shapes: InteractionResult[]) => ( - // converts output of cvat-canvas to server input - { - points: shapes.filter((shape: InteractionResult): boolean => shape.shapeType === 'points') - .map((shape: InteractionResult): number[] => shape.points) - .flat().reduce(reducer, []), - } - ), -}; diff --git a/cvat-ui/src/utils/plugin-checker.ts b/cvat-ui/src/utils/plugin-checker.ts index 38c29575e793..e41506106936 100644 --- a/cvat-ui/src/utils/plugin-checker.ts +++ b/cvat-ui/src/utils/plugin-checker.ts @@ -17,14 +17,6 @@ class PluginChecker { case SupportedPlugins.GIT_INTEGRATION: { return isReachable(`${serverHost}/git/repository/meta/get`, 'OPTIONS'); } - case SupportedPlugins.DEXTR_SEGMENTATION: { - try { - const list = await core.lambda.list(); - return list.map((func: any): boolean => func.id).includes('openvino.dextr'); - } catch (_) { - return false; - } - } case SupportedPlugins.ANALYTICS: { return isReachable(`${serverHost}/analytics/app/kibana`, 'GET'); } From 199c2644444b2412ca8ab237a7102a8d4910de22 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 19 Aug 2020 17:42:53 +0300 Subject: [PATCH 04/26] Updated icon: magic wand --- cvat-ui/src/assets/ai-tools.svg | 18 ++++++++++++++++++ .../controls-side-bar/tools-control.tsx | 10 ++-------- cvat-ui/src/icons.tsx | 4 ++++ 3 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 cvat-ui/src/assets/ai-tools.svg diff --git a/cvat-ui/src/assets/ai-tools.svg b/cvat-ui/src/assets/ai-tools.svg new file mode 100644 index 000000000000..c8b4f304b8d4 --- /dev/null +++ b/cvat-ui/src/assets/ai-tools.svg @@ -0,0 +1,18 @@ + + + + + diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index c31203f7f09a..f9fcb7bf97ad 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -12,6 +12,7 @@ import Text from 'antd/lib/typography/Text'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; +import { AITools } from 'icons'; import { Canvas } from 'cvat-canvas-wrapper'; import getCore from 'cvat-core-wrapper'; import { @@ -118,13 +119,6 @@ class ToolsControlComponent extends React.PureComponent { } public componentDidUpdate(prevProps: Props): void { - const icon = window.document.getElementsByClassName('cvat-tools-control')[0]; - if (icon) { - const svg: SVGSVGElement = icon.children[0] as SVGSVGElement; - svg.setAttribute('width', '40px'); - svg.setAttribute('height', '40px'); - } - const { isInteraction, jobInstance } = this.props; const { interactiveStateID } = this.state; if (prevProps.isInteraction && !isInteraction) { @@ -365,7 +359,7 @@ class ToolsControlComponent extends React.PureComponent { overlayClassName='cvat-tools-control-popover' content={interactors.length && this.renderPopoverContent()} > - + ); } diff --git a/cvat-ui/src/icons.tsx b/cvat-ui/src/icons.tsx index 1eed125bf327..d4fccc49dbdc 100644 --- a/cvat-ui/src/icons.tsx +++ b/cvat-ui/src/icons.tsx @@ -41,6 +41,7 @@ import SVGBackgroundIcon from './assets/background-icon.svg'; import SVGForegroundIcon from './assets/foreground-icon.svg'; import SVGCubeIcon from './assets/cube-icon.svg'; import SVGResetPerspectiveIcon from './assets/reset-perspective.svg'; +import SVGAITools from './assets/ai-tools.svg'; export const CVATLogo = React.memo( (): JSX.Element => , @@ -153,3 +154,6 @@ export const CubeIcon = React.memo( export const ResetPerspectiveIcon = React.memo( (): JSX.Element => , ); +export const AITools = React.memo( + (): JSX.Element => , +); From c3132d11ed8015816cc677dbb133d7391585809d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Wed, 19 Aug 2020 18:30:00 +0300 Subject: [PATCH 05/26] Ctrl modifier, fixed some cases when interaction event isn't raised --- cvat-canvas/README.md | 7 +-- cvat-canvas/src/typescript/canvasModel.ts | 2 +- cvat-canvas/src/typescript/canvasView.ts | 5 ++ cvat-canvas/src/typescript/crosshair.ts | 2 - .../src/typescript/interactionHandler.ts | 52 ++++++++++++------- .../controls-side-bar/tools-control.tsx | 2 +- 6 files changed, 40 insertions(+), 30 deletions(-) diff --git a/cvat-canvas/README.md b/cvat-canvas/README.md index cbc7fd324fd5..d6a5698bae41 100644 --- a/cvat-canvas/README.md +++ b/cvat-canvas/README.md @@ -73,12 +73,7 @@ Canvas itself handles: interface InteractionData { shapeType: string; - numberOfShapes?: number; - mouse: { - right: 'positive' | 'negative' | 'cancel'; - left: 'positive' | 'negative' | 'cancel'; - center: 'positive' | 'negative' | 'cancel'; - }; + minVertices?: number; } interface GroupData { diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index 80749fc1e87a..9969ce1b0299 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -73,7 +73,7 @@ export interface InteractionData { enabled: boolean; shapeType?: string; crosshair?: boolean; - numberOfShapes?: number; + minVertices?: number; result?: 'immediate' | 'deferred'; } diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts index 26244dec3f37..13dc922e02c1 100644 --- a/cvat-canvas/src/typescript/canvasView.ts +++ b/cvat-canvas/src/typescript/canvasView.ts @@ -1462,6 +1462,11 @@ export class CanvasViewImpl implements CanvasView, Listener { [state, +state.getAttribute('data-z-order')] )); + const crosshair = Array.from(this.content.getElementsByClassName('cvat_canvas_crosshair')); + crosshair.forEach((line: SVGLineElement): void => this.content.append(line)); + const interaction = Array.from(this.content.getElementsByClassName('cvat_interaction_point')); + interaction.forEach((circle: SVGCircleElement): void => this.content.append(circle)); + const needSort = states.some((pair): boolean => pair[1] !== states[0][1]); if (!states.length || !needSort) { return; diff --git a/cvat-canvas/src/typescript/crosshair.ts b/cvat-canvas/src/typescript/crosshair.ts index c55698f36a18..dde16ccce177 100644 --- a/cvat-canvas/src/typescript/crosshair.ts +++ b/cvat-canvas/src/typescript/crosshair.ts @@ -27,12 +27,10 @@ export default class Crosshair { this.canvas = canvas; this.x = this.canvas.line(0, y, this.canvas.node.clientWidth, y).attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * scale), - zOrder: Number.MAX_SAFE_INTEGER, }).addClass('cvat_canvas_crosshair'); this.y = this.canvas.line(x, 0, x, this.canvas.node.clientHeight).attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / (2 * scale), - zOrder: Number.MAX_SAFE_INTEGER, }).addClass('cvat_canvas_crosshair'); } diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts index b5e67579e582..6574a7bd5009 100644 --- a/cvat-canvas/src/typescript/interactionHandler.ts +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -21,6 +21,7 @@ export class InteractionHandlerImpl implements InteractionHandler { private canvas: SVG.Container; private interactionData: InteractionData; private cursorPosition: { x: number; y: number }; + private shapesWereUpdated: boolean; private interactionShapes: SVG.Shape[]; private currentInteractionShape: SVG.Shape | null; private crosshair: Crosshair; @@ -46,6 +47,15 @@ export class InteractionHandlerImpl implements InteractionHandler { }); } + private shouldRaiseEvent(ctrlKey: boolean): boolean { + const { interactionData, interactionShapes, shapesWereUpdated } = this; + const { minVertices, enabled } = interactionData; + + const minimumVerticesAchieved = typeof (minVertices) === 'undefined' + || minVertices <= interactionShapes.length; + return enabled && !ctrlKey && minimumVerticesAchieved && shapesWereUpdated; + } + private addCrosshair(): void { const { x, y } = this.cursorPosition; this.crosshair.show(this.canvas, x, y, this.geometry.scale); @@ -69,32 +79,27 @@ export class InteractionHandlerImpl implements InteractionHandler { }); this.interactionShapes.push(this.currentInteractionShape); - if (this.interactionData.result === 'immediate') { + this.shapesWereUpdated = true; + if (this.shouldRaiseEvent(e.ctrlKey)) { this.onInteraction(this.prepareResult()); } - if (typeof (this.interactionData.numberOfShapes) !== 'undefined' - && this.interactionShapes.length >= this.interactionData.numberOfShapes) { - if (this.interactionData.result !== 'immediate') { - this.onInteraction(this.prepareResult()); - } - - this.interact({ enabled: false }); - return; - } - const self = this.currentInteractionShape; self.on('mouseenter', (): void => { self.attr({ 'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / this.geometry.scale, }); - self.on('mousedown', (e: MouseEvent) => { - e.stopPropagation(); + self.on('mousedown', (_e: MouseEvent) => { + _e.stopPropagation(); self.remove(); this.interactionShapes = this.interactionShapes.filter( (shape: SVG.Shape): boolean => shape !== self ); + this.shapesWereUpdated = true; + if (this.shouldRaiseEvent(_e.ctrlKey)) { + this.onInteraction(this.prepareResult()); + } }); }); @@ -129,18 +134,14 @@ export class InteractionHandlerImpl implements InteractionHandler { this.canvas.on('mousedown.interaction', eventListener); this.currentInteractionShape.on('drawstop', (): void => { this.interactionShapes.push(this.currentInteractionShape); + this.shapesWereUpdated = true; this.canvas.off('mousedown.interaction', eventListener); if (this.interactionData.result === 'immediate') { this.onInteraction(this.prepareResult()); } - if (typeof (this.interactionData.numberOfShapes) === 'undefined' - || this.interactionShapes.length < this.interactionData.numberOfShapes) { - this.interactRectangle(); - } else { - this.interact({ enabled: false }); - } + this.interact({ enabled: false }); }).addClass('cvat_canvas_shape_drawing').attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, }); @@ -184,11 +185,16 @@ export class InteractionHandlerImpl implements InteractionHandler { canvas: SVG.Container, geometry: Geometry, ) { - this.onInteraction = onInteraction; + this.onInteraction = (shapes: InteractionResult[] | null) => { + this.shapesWereUpdated = false; + onInteraction(shapes); + }; this.onStopInteraction = onStopInteraction; this.canvas = canvas; this.geometry = geometry; + this.shapesWereUpdated = false; this.interactionShapes = []; + this.interactionData = { enabled: false }; this.currentInteractionShape = null; this.crosshair = new Crosshair(); this.cursorPosition = { @@ -206,6 +212,12 @@ export class InteractionHandlerImpl implements InteractionHandler { this.crosshair.move(x, y); } }); + + document.body.addEventListener('keyup', (e: KeyboardEvent): void => { + if (e.keyCode === 17 && this.shouldRaiseEvent(false)) { // 17 is ctrl + this.onInteraction(this.prepareResult()); + } + }); } public transform(geometry: Geometry): void { diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index f9fcb7bf97ad..57ff950e34ad 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -303,7 +303,7 @@ class ToolsControlComponent extends React.PureComponent { canvasInstance.cancel(); canvasInstance.interact({ shapeType: 'points', - numberOfShapes: 4, // TODO: Add parameter to server + minVertices: 4, // TODO: Add parameter to interactor enabled: true, }); From c9dab60b35cce849ef6ac2ae1caf3c9993c5a6ed Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 20 Aug 2020 11:59:34 +0300 Subject: [PATCH 06/26] Added tooltip description of an interactor --- .../standard-workspace/controls-side-bar/tools-control.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 57ff950e34ad..e4dd7aa7b326 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -284,6 +284,7 @@ class ToolsControlComponent extends React.PureComponent { > {interactors.map((interactor: Model): JSX.Element => ( {interactor.name} From 2ad4ae62483d16e205cb2ad8c02af15b3c9f6abc Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 20 Aug 2020 12:55:30 +0300 Subject: [PATCH 07/26] Locking UI while server fetching --- cvat-canvas/src/typescript/canvasModel.ts | 1 - .../src/typescript/interactionHandler.ts | 4 +-- .../controls-side-bar/tools-control.tsx | 31 ++++++++++++++----- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index 9969ce1b0299..3b497f25f4f8 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -74,7 +74,6 @@ export interface InteractionData { shapeType?: string; crosshair?: boolean; minVertices?: number; - result?: 'immediate' | 'deferred'; } export interface InteractionResult { diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts index 6574a7bd5009..b0c77103a0ab 100644 --- a/cvat-canvas/src/typescript/interactionHandler.ts +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -137,7 +137,7 @@ export class InteractionHandlerImpl implements InteractionHandler { this.shapesWereUpdated = true; this.canvas.off('mousedown.interaction', eventListener); - if (this.interactionData.result === 'immediate') { + if (this.shouldRaiseEvent(false)) { this.onInteraction(this.prepareResult()); } @@ -245,7 +245,7 @@ export class InteractionHandlerImpl implements InteractionHandler { this.initInteraction(); this.startInteraction(); } else { - if (this.interactionData.result !== 'immediate') { + if (this.shouldRaiseEvent(false)) { this.onInteraction(this.prepareResult()); } diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index e4dd7aa7b326..5ac2cbb23bf4 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -8,6 +8,7 @@ import Icon from 'antd/lib/icon'; import Popover from 'antd/lib/popover'; import Select, { OptionProps } from 'antd/lib/select'; import Button from 'antd/lib/button'; +import Modal from 'antd/lib/modal'; import Text from 'antd/lib/typography/Text'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; @@ -336,6 +337,8 @@ class ToolsControlComponent extends React.PureComponent { public render(): JSX.Element | null { const { interactors, isInteraction, canvasInstance } = this.props; + const { fetching } = this.state; + if (!interactors.length) return null; const dynamcPopoverPros = isInteraction ? { @@ -354,14 +357,26 @@ class ToolsControlComponent extends React.PureComponent { }; return ( - - - + <> + + Waiting for a server response.. + + + + + + ); } } From c9fd42ed5ddde6114d0054c1bde6bd7910d4aed4 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 20 Aug 2020 13:13:53 +0300 Subject: [PATCH 08/26] Removing old code & refactoring --- cvat-ui/src/actions/models-actions.ts | 6 --- .../model-runner-modal/model-runner-modal.tsx | 46 +++++++---------- .../models-page/built-model-item.tsx | 49 ------------------- .../models-page/deployed-models-list.tsx | 1 - .../components/models-page/models-page.tsx | 14 +++++- .../model-runner-dialog.tsx | 6 ++- .../containers/models-page/models-page.tsx | 16 +++++- cvat-ui/src/reducers/interfaces.ts | 6 +-- cvat-ui/src/reducers/models-reducer.ts | 8 ++- cvat-ui/src/reducers/notifications-reducer.ts | 17 ------- 10 files changed, 56 insertions(+), 113 deletions(-) delete mode 100644 cvat-ui/src/components/models-page/built-model-item.tsx diff --git a/cvat-ui/src/actions/models-actions.ts b/cvat-ui/src/actions/models-actions.ts index 8bc164961b49..2627e5ffa786 100644 --- a/cvat-ui/src/actions/models-actions.ts +++ b/cvat-ui/src/actions/models-actions.ts @@ -10,11 +10,6 @@ export enum ModelsActionTypes { GET_MODELS = 'GET_MODELS', GET_MODELS_SUCCESS = 'GET_MODELS_SUCCESS', GET_MODELS_FAILED = 'GET_MODELS_FAILED', - DELETE_MODEL = 'DELETE_MODEL', - CREATE_MODEL = 'CREATE_MODEL', - CREATE_MODEL_SUCCESS = 'CREATE_MODEL_SUCCESS', - CREATE_MODEL_FAILED = 'CREATE_MODEL_FAILED', - CREATE_MODEL_STATUS_UPDATED = 'CREATE_MODEL_STATUS_UPDATED', START_INFERENCE_FAILED = 'START_INFERENCE_FAILED', GET_INFERENCE_STATUS_SUCCESS = 'GET_INFERENCE_STATUS_SUCCESS', GET_INFERENCE_STATUS_FAILED = 'GET_INFERENCE_STATUS_FAILED', @@ -161,7 +156,6 @@ export function startInferenceAsync( return async (dispatch): Promise => { try { const requestID: string = await core.lambda.run(taskInstance, model, body); - const dispatchCallback = (action: ModelsActions): void => { dispatch(action); }; diff --git a/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx b/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx index 234c7fd0850c..d8fd959ecb96 100644 --- a/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx +++ b/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx @@ -21,7 +21,8 @@ import { } from 'reducers/interfaces'; interface Props { - models: Model[]; + reid: Model[]; + detectors: Model[]; activeProcesses: StringObject; visible: boolean; taskInstance: any; @@ -88,14 +89,14 @@ export default class ModelRunnerModalComponent extends React.PureComponent @@ -166,10 +168,7 @@ export default class ModelRunnerModalComponent extends React.PureComponent @@ -203,11 +202,7 @@ export default class ModelRunnerModalComponent extends React.PureComponent @@ -346,16 +338,10 @@ export default class ModelRunnerModalComponent extends React.PureComponent _model.name === selectedModel)[0]; @@ -414,13 +400,15 @@ export default class ModelRunnerModalComponent extends React.PureComponent model.name === selectedModel, )[0]; diff --git a/cvat-ui/src/components/models-page/built-model-item.tsx b/cvat-ui/src/components/models-page/built-model-item.tsx deleted file mode 100644 index f100dfe5a4e0..000000000000 --- a/cvat-ui/src/components/models-page/built-model-item.tsx +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import React from 'react'; -import { Row, Col } from 'antd/lib/grid'; -import Tag from 'antd/lib/tag'; -import Select from 'antd/lib/select'; -import Text from 'antd/lib/typography/Text'; - -import { Model } from 'reducers/interfaces'; - -interface Props { - model: Model; -} - -export default function BuiltModelItemComponent(props: Props): JSX.Element { - const { model } = props; - - return ( - - - {model.framework} - - - - {model.name} - - - - - - - - ); -} diff --git a/cvat-ui/src/components/models-page/deployed-models-list.tsx b/cvat-ui/src/components/models-page/deployed-models-list.tsx index 93e301be2ef0..45d62aa87b1d 100644 --- a/cvat-ui/src/components/models-page/deployed-models-list.tsx +++ b/cvat-ui/src/components/models-page/deployed-models-list.tsx @@ -9,7 +9,6 @@ import Text from 'antd/lib/typography/Text'; import { Model } from 'reducers/interfaces'; import DeployedModelItem from './deployed-model-item'; - interface Props { models: Model[]; } diff --git a/cvat-ui/src/components/models-page/models-page.tsx b/cvat-ui/src/components/models-page/models-page.tsx index a5606a2ade15..3b4fe521c516 100644 --- a/cvat-ui/src/components/models-page/models-page.tsx +++ b/cvat-ui/src/components/models-page/models-page.tsx @@ -12,11 +12,21 @@ import FeedbackComponent from '../feedback/feedback'; import { Model } from '../../reducers/interfaces'; interface Props { - deployedModels: Model[]; + interactors: Model[]; + detectors: Model[]; + trackers: Model[]; + reid: Model[]; } export default function ModelsPageComponent(props: Props): JSX.Element { - const { deployedModels } = props; + const { + interactors, + detectors, + trackers, + reid, + } = props; + + const deployedModels = [...detectors, ...interactors, ...trackers, ...reid]; return (
diff --git a/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx b/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx index b956876ca529..3f63bd7f6a11 100644 --- a/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx +++ b/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx @@ -9,7 +9,8 @@ import { Model, CombinedState } from 'reducers/interfaces'; import { startInferenceAsync, modelsActions } from 'actions/models-actions'; interface StateToProps { - models: Model[]; + reid: Model[]; + detectors: Model[]; activeProcesses: { [index: string]: string; }; @@ -30,7 +31,8 @@ function mapStateToProps(state: CombinedState): StateToProps { const { models } = state; return { - models: models.models, + reid: models.reid, + detectors: models.detectors, activeProcesses: {}, taskInstance: models.activeRunTask, visible: models.visibleRunWindows, diff --git a/cvat-ui/src/containers/models-page/models-page.tsx b/cvat-ui/src/containers/models-page/models-page.tsx index 8282d41db42f..734ad0dcbce2 100644 --- a/cvat-ui/src/containers/models-page/models-page.tsx +++ b/cvat-ui/src/containers/models-page/models-page.tsx @@ -11,14 +11,26 @@ import { } from 'reducers/interfaces'; interface StateToProps { - deployedModels: Model[]; + interactors: Model[]; + detectors: Model[]; + trackers: Model[]; + reid: Model[]; } function mapStateToProps(state: CombinedState): StateToProps { const { models } = state; + const { + interactors, + detectors, + trackers, + reid, + } = models; return { - deployedModels: models.models, + interactors, + detectors, + trackers, + reid, }; } diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index ec8138189a1e..8e0deb419cff 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -159,8 +159,10 @@ export interface ModelsState { initialized: boolean; fetching: boolean; creatingStatus: string; - models: Model[]; interactors: Model[]; + detectors: Model[]; + trackers: Model[]; + reid: Model[]; inferences: { [index: number]: ActiveInference; }; @@ -205,9 +207,7 @@ export interface NotificationsState { fetching: null | ErrorState; }; models: { - creating: null | ErrorState; starting: null | ErrorState; - deleting: null | ErrorState; fetching: null | ErrorState; canceling: null | ErrorState; metaFetching: null | ErrorState; diff --git a/cvat-ui/src/reducers/models-reducer.ts b/cvat-ui/src/reducers/models-reducer.ts index 26a9775340be..18371aea02bd 100644 --- a/cvat-ui/src/reducers/models-reducer.ts +++ b/cvat-ui/src/reducers/models-reducer.ts @@ -11,8 +11,10 @@ const defaultState: ModelsState = { initialized: false, fetching: false, creatingStatus: '', - models: [], interactors: [], + detectors: [], + trackers: [], + reid: [], visibleRunWindows: false, activeRunTask: null, inferences: {}, @@ -33,8 +35,10 @@ export default function ( case ModelsActionTypes.GET_MODELS_SUCCESS: { return { ...state, - models: action.payload.models.filter((model: Model) => ['detector', 'reid'].includes(model.type)), interactors: action.payload.models.filter((model: Model) => ['interactor'].includes(model.type)), + detectors: action.payload.models.filter((model: Model) => ['detector'].includes(model.type)), + trackers: action.payload.models.filter((model: Model) => ['tracker'].includes(model.type)), + reid: action.payload.models.filter((model: Model) => ['reid'].includes(model.type)), initialized: true, fetching: false, }; diff --git a/cvat-ui/src/reducers/notifications-reducer.ts b/cvat-ui/src/reducers/notifications-reducer.ts index e4da70868d95..43a51f0d1443 100644 --- a/cvat-ui/src/reducers/notifications-reducer.ts +++ b/cvat-ui/src/reducers/notifications-reducer.ts @@ -51,9 +51,7 @@ const defaultState: NotificationsState = { fetching: null, }, models: { - creating: null, starting: null, - deleting: null, fetching: null, canceling: null, metaFetching: null, @@ -394,21 +392,6 @@ export default function (state = defaultState, action: AnyAction): Notifications }, }; } - case ModelsActionTypes.CREATE_MODEL_FAILED: { - return { - ...state, - errors: { - ...state.errors, - models: { - ...state.errors.models, - creating: { - message: 'Could not create the model', - reason: action.payload.error.toString(), - }, - }, - }, - }; - } case ModelsActionTypes.GET_INFERENCE_STATUS_SUCCESS: { if (action.payload.activeInference.status === 'finished') { const { taskID } = action.payload; From e47709289a6f744a463bb6bff24ebae1f15d704b Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 20 Aug 2020 13:59:58 +0300 Subject: [PATCH 09/26] Fixed couple of bugs --- cvat-canvas/src/typescript/canvasModel.ts | 2 +- cvat-ui/src/actions/annotation-actions.ts | 6 ++- .../controls-side-bar/tools-control.tsx | 37 +++++++------------ 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index 3b497f25f4f8..a2ae93802b92 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -606,7 +606,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { } public isAbleToChangeFrame(): boolean { - const isUnable = [Mode.DRAG, Mode.EDIT, Mode.RESIZE].includes(this.data.mode) + const isUnable = [Mode.DRAG, Mode.EDIT, Mode.RESIZE, Mode.INTERACT].includes(this.data.mode) || (this.data.mode === Mode.DRAW && typeof (this.data.drawData.redraw) === 'number'); return !isUnable; diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 430566de5ff2..cf8dee9a25be 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -1464,7 +1464,11 @@ export function repeatDrawShapeAsync(): ThunkAction { let activeControl = ActiveControl.CURSOR; if (activeInteractor) { - canvasInstance.interact({ enabled: true }); + canvasInstance.interact({ + enabled: true, + shapeType: 'points', + minVertices: 4, // TODO: Add parameter to interactor + }); dispatch(interactWithCanvas(activeInteractor, activeLabelID)); return; } diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 5ac2cbb23bf4..7837398ae6e7 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -125,9 +125,7 @@ class ToolsControlComponent extends React.PureComponent { if (prevProps.isInteraction && !isInteraction) { if (interactiveStateID !== null) { jobInstance.actions.freeze(false); - this.setState({ - interactiveStateID: null, - }); + this.setState({ interactiveStateID: null }); } } } @@ -151,9 +149,7 @@ class ToolsControlComponent extends React.PureComponent { const { activeInteractor, interactiveStateID } = this.state; try { - this.setState({ - fetching: true, - }); + this.setState({ fetching: true }); if (!isInteraction) { throw Error('Canvas raises "canvas.interacted" when interaction is off'); @@ -166,6 +162,7 @@ class ToolsControlComponent extends React.PureComponent { points: convertShapesForInteractor((e as CustomEvent).detail.shapes), }); + // no shape yet, then create it and save to collection if (interactiveStateID === null) { const object = new core.classes.ObjectState({ frame, @@ -177,19 +174,21 @@ class ToolsControlComponent extends React.PureComponent { occluded: false, zOrder: (e as CustomEvent).detail.zOrder, }); + // need a clientID of a created object, so, we do not use createAnnotationAction const [clientID] = await jobInstance.annotations.put([object]); + + // update annotations on a canvas fetchAnnotations(); - await jobInstance.actions.freeze(true); // only if still interaction - this.setState({ - interactiveStateID: clientID, - }); + + // freeze history for interaction time + // (points updating shouldn't cause adding new actions to history) + await jobInstance.actions.freeze(true); + this.setState({ interactiveStateID: clientID }); } else { const state = states .filter((_state: any): boolean => _state.clientID === interactiveStateID)[0]; - state.points = result.flat(); await updateAnnotations([state]); - // TODO: disable switching between frames } } catch (err) { notification.error({ @@ -197,9 +196,7 @@ class ToolsControlComponent extends React.PureComponent { message: 'Interaction error occured', }); } finally { - this.setState({ - fetching: false, - }); + this.setState({ fetching: false }); } }; @@ -250,10 +247,7 @@ class ToolsControlComponent extends React.PureComponent { > { labels.map((label: any) => ( - + {label.name} )) @@ -284,10 +278,7 @@ class ToolsControlComponent extends React.PureComponent { onChange={this.setActiveInteractor} > {interactors.map((interactor: Model): JSX.Element => ( - + {interactor.name} ))} From 6b17687449654a9999749d19004550f7908d2a23 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 20 Aug 2020 14:02:27 +0300 Subject: [PATCH 10/26] Updated CHANGELOG.md, updated versions --- CHANGELOG.md | 1 + cvat-canvas/package-lock.json | 2 +- cvat-canvas/package.json | 2 +- cvat-core/package-lock.json | 2 +- cvat-core/package.json | 2 +- cvat-ui/package-lock.json | 2 +- cvat-ui/package.json | 2 +- 7 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26e3bafcdd66..2ebbcf430974 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Shape coordinates are rounded to 2 digits in dumped annotations () - COCO format does not produce polygon points for bbox annotations () +- UI models (like DEXTR) were redesigned to be more interactive () ### Deprecated - diff --git a/cvat-canvas/package-lock.json b/cvat-canvas/package-lock.json index b50b6223aa66..5ee1996827da 100644 --- a/cvat-canvas/package-lock.json +++ b/cvat-canvas/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-canvas", - "version": "2.0.2", + "version": "2.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json index 8792ce080236..ba7f0180f584 100644 --- a/cvat-canvas/package.json +++ b/cvat-canvas/package.json @@ -1,6 +1,6 @@ { "name": "cvat-canvas", - "version": "2.0.2", + "version": "2.1.0", "description": "Part of Computer Vision Annotation Tool which presents its canvas library", "main": "src/canvas.ts", "scripts": { diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index d79d54d0a649..0bc2c78700cf 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.3.1", + "version": "3.5.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index 7d75dbc525e8..ca65f045f7d4 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.4.0", + "version": "3.5.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index 8def8aaa4709..f8f63da46401 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.7.1", + "version": "1.8.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index f20e3eb6da67..d47c1da670be 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.7.1", + "version": "1.8.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { From 61f710453c2b65815799a775a881747eba92ae0e Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 20 Aug 2020 14:05:05 +0300 Subject: [PATCH 11/26] Update crosshair.ts --- cvat-canvas/src/typescript/crosshair.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-canvas/src/typescript/crosshair.ts b/cvat-canvas/src/typescript/crosshair.ts index dde16ccce177..2d160137fec8 100644 --- a/cvat-canvas/src/typescript/crosshair.ts +++ b/cvat-canvas/src/typescript/crosshair.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2020 Intel Corporation // // SPDX-License-Identifier: MIT From 057f41ab2a60939bdede5b5243954a50286d520d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 21 Aug 2020 17:52:11 +0300 Subject: [PATCH 12/26] Minor fixes --- .../{ai-tools.svg => ai-tools-icon.svg} | 0 .../controls-side-bar/tools-control.tsx | 20 +++++++------------ cvat-ui/src/icons.tsx | 4 ++-- 3 files changed, 9 insertions(+), 15 deletions(-) rename cvat-ui/src/assets/{ai-tools.svg => ai-tools-icon.svg} (100%) diff --git a/cvat-ui/src/assets/ai-tools.svg b/cvat-ui/src/assets/ai-tools-icon.svg similarity index 100% rename from cvat-ui/src/assets/ai-tools.svg rename to cvat-ui/src/assets/ai-tools-icon.svg diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 7837398ae6e7..74c365bf0ff9 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -13,7 +13,7 @@ import Text from 'antd/lib/typography/Text'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; -import { AITools } from 'icons'; +import { AIToolsIcon } from 'icons'; import { Canvas } from 'cvat-canvas-wrapper'; import getCore from 'cvat-core-wrapper'; import { @@ -209,17 +209,9 @@ class ToolsControlComponent extends React.PureComponent { }); }; - private setActiveLabel = (key: string): void => { - const { labels } = this.props; - this.setState({ - activeLabelID: labels.filter( - (label: any) => label.id === +key, - )[0], - }); - }; - private renderLabelBlock(): JSX.Element { - const { labels, activeLabelID } = this.props; + const { labels } = this.props; + const { activeLabelID } = this.state; return ( <> @@ -243,7 +235,9 @@ class ToolsControlComponent extends React.PureComponent { } } value={`${activeLabelID}`} - onChange={this.setActiveLabel} + onChange={(value: string) => { + this.setState({ activeLabelID: +value }); + }} > { labels.map((label: any) => ( @@ -365,7 +359,7 @@ class ToolsControlComponent extends React.PureComponent { overlayClassName='cvat-tools-control-popover' content={interactors.length && this.renderPopoverContent()} > - + ); diff --git a/cvat-ui/src/icons.tsx b/cvat-ui/src/icons.tsx index 0a0c83643097..a3e92e929aef 100644 --- a/cvat-ui/src/icons.tsx +++ b/cvat-ui/src/icons.tsx @@ -42,7 +42,7 @@ import SVGForegroundIcon from './assets/foreground-icon.svg'; import SVGCubeIcon from './assets/cube-icon.svg'; import SVGResetPerspectiveIcon from './assets/reset-perspective.svg'; import SVGColorizeIcon from './assets/colorize-icon.svg'; -import SVGAITools from './assets/ai-tools.svg'; +import SVGAITools from './assets/ai-tools-icon.svg'; export const CVATLogo = React.memo( @@ -156,7 +156,7 @@ export const CubeIcon = React.memo( export const ResetPerspectiveIcon = React.memo( (): JSX.Element => , ); -export const AITools = React.memo( +export const AIToolsIcon = React.memo( (): JSX.Element => , ); export const ColorizeIcon = React.memo( From e75738378b34ca9a07733a5384aa2526c7eb90f8 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 21 Aug 2020 18:03:28 +0300 Subject: [PATCH 13/26] Fixed eslint issues --- cvat-canvas/src/typescript/canvasModel.ts | 2 +- cvat-canvas/src/typescript/crosshair.ts | 10 +++--- cvat-canvas/src/typescript/editHandler.ts | 12 +++---- .../src/typescript/interactionHandler.ts | 34 +++++++++++-------- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index a2ae93802b92..8f171483660d 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -80,7 +80,7 @@ export interface InteractionResult { points: number[]; shapeType: string; button: number; -}; +} export interface EditData { enabled: boolean; diff --git a/cvat-canvas/src/typescript/crosshair.ts b/cvat-canvas/src/typescript/crosshair.ts index 2d160137fec8..27d255698ac8 100644 --- a/cvat-canvas/src/typescript/crosshair.ts +++ b/cvat-canvas/src/typescript/crosshair.ts @@ -10,13 +10,13 @@ export default class Crosshair { private y: SVG.Line | null; private canvas: SVG.Container | null; - constructor() { + public constructor() { this.x = null; this.y = null; this.canvas = null; } - show(canvas: SVG.Container, x: number, y: number, scale: number) { + public show(canvas: SVG.Container, x: number, y: number, scale: number): void { if (this.canvas && this.canvas !== canvas) { if (this.x) this.x.remove(); if (this.y) this.y.remove(); @@ -34,7 +34,7 @@ export default class Crosshair { }).addClass('cvat_canvas_crosshair'); } - hide() { + public hide(): void { if (this.x) { this.x.remove(); this.x = null; @@ -48,7 +48,7 @@ export default class Crosshair { this.canvas = null; } - move(x: number, y: number) { + public move(x: number, y: number): void { if (this.x) { this.x.attr({ y1: y, y2: y }); } @@ -58,7 +58,7 @@ export default class Crosshair { } } - scale(scale: number) { + public scale(scale: number): void { if (this.x) { this.x.attr('stroke-width', consts.BASE_STROKE_WIDTH / (2 * scale)); } diff --git a/cvat-canvas/src/typescript/editHandler.ts b/cvat-canvas/src/typescript/editHandler.ts index 8cea079d148b..413beda73665 100644 --- a/cvat-canvas/src/typescript/editHandler.ts +++ b/cvat-canvas/src/typescript/editHandler.ts @@ -115,7 +115,7 @@ export class EditHandlerImpl implements EditHandler { (this.editLine as any).addClass('cvat_canvas_shape_drawing').style({ 'pointer-events': 'none', 'fill-opacity': 0, - 'stroke': strokeColor, + stroke: strokeColor, }).attr({ 'data-origin-client-id': this.editData.state.clientID, }).on('drawstart drawpoint', (e: CustomEvent): void => { @@ -213,20 +213,20 @@ export class EditHandlerImpl implements EditHandler { const cutIndexes2 = oldPoints.reduce((acc: string[], _: string, i: number) => i <= stop && i >= start ? [...acc, i] : acc, []); - const curveLength = (indexes: number[]) => { + const curveLength = (indexes: number[]): number => { const points = indexes.map((index: number): string => oldPoints[index]) .map((point: string): string[] => point.split(',')) .map((point: string[]): number[] => [+point[0], +point[1]]); let length = 0; for (let i = 1; i < points.length; i++) { length += Math.sqrt( - (points[i][0] - points[i - 1][0]) ** 2 - + (points[i][1] - points[i - 1][1]) ** 2, + ((points[i][0] - points[i - 1][0]) ** 2) + + ((points[i][1] - points[i - 1][1]) ** 2), ); } return length; - } + }; const pointsCriteria = cutIndexes1.length > cutIndexes2.length; const lengthCriteria = curveLength(cutIndexes1) > curveLength(cutIndexes2); @@ -278,8 +278,6 @@ export class EditHandlerImpl implements EditHandler { }); } } - - return; } private setupPoints(enabled: boolean): void { diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts index b0c77103a0ab..89f1642742af 100644 --- a/cvat-canvas/src/typescript/interactionHandler.ts +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -16,7 +16,7 @@ export interface InteractionHandler { export class InteractionHandlerImpl implements InteractionHandler { private onInteraction: (shapes: InteractionResult[] | null) => void; - onStopInteraction: () => void; + private onStopInteraction: () => void; private geometry: Geometry; private canvas: SVG.Container; private interactionData: InteractionData; @@ -35,15 +35,15 @@ export class InteractionHandlerImpl implements InteractionHandler { shapeType: 'points', button: (shape.style('stroke') as any as string) === 'red' ? 2 : 0, }; - } else { - const bbox = (shape.node as any as SVGRectElement).getBBox(); - const points = [bbox.x, bbox.y, bbox.x + bbox.width, bbox.y + bbox.height]; - return { - points: points.map((coord: number): number => coord - this.geometry.offset), - shapeType: 'rectangle', - button: 0, - }; } + + const bbox = (shape.node as any as SVGRectElement).getBBox(); + const points = [bbox.x, bbox.y, bbox.x + bbox.width, bbox.y + bbox.height]; + return { + points: points.map((coord: number): number => coord - this.geometry.offset), + shapeType: 'rectangle', + button: 0, + }; }); } @@ -68,7 +68,10 @@ export class InteractionHandlerImpl implements InteractionHandler { private interactPoints(): void { const eventListener = (e: MouseEvent): void => { if ((e.button === 0 || e.button === 2) && !e.altKey) { - const [cx, cy] = translateToSVG(this.canvas.node as any as SVGSVGElement, [e.clientX, e.clientY]); + const [cx, cy] = translateToSVG( + this.canvas.node as any as SVGSVGElement, + [e.clientX, e.clientY], + ); this.currentInteractionShape = this.canvas .circle(consts.BASE_POINT_SIZE * 2 / this.geometry.scale).center(cx, cy) .fill('white') @@ -90,11 +93,11 @@ export class InteractionHandlerImpl implements InteractionHandler { 'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / this.geometry.scale, }); - self.on('mousedown', (_e: MouseEvent) => { + self.on('mousedown', (_e: MouseEvent): void => { _e.stopPropagation(); self.remove(); this.interactionShapes = this.interactionShapes.filter( - (shape: SVG.Shape): boolean => shape !== self + (shape: SVG.Shape): boolean => shape !== self, ); this.shapesWereUpdated = true; if (this.shouldRaiseEvent(_e.ctrlKey)) { @@ -185,7 +188,7 @@ export class InteractionHandlerImpl implements InteractionHandler { canvas: SVG.Container, geometry: Geometry, ) { - this.onInteraction = (shapes: InteractionResult[] | null) => { + this.onInteraction = (shapes: InteractionResult[] | null): void => { this.shapesWereUpdated = false; onInteraction(shapes); }; @@ -227,8 +230,9 @@ export class InteractionHandlerImpl implements InteractionHandler { this.crosshair.scale(this.geometry.scale); } - const shapesToBeScaled = this.currentInteractionShape ? - [...this.interactionShapes, this.currentInteractionShape] : [...this.interactionShapes]; + const shapesToBeScaled = this.currentInteractionShape + ? [...this.interactionShapes, this.currentInteractionShape] + : [...this.interactionShapes]; for (const shape of shapesToBeScaled) { if (shape.type === 'circle') { (shape as SVG.Circle).radius(consts.BASE_POINT_SIZE / this.geometry.scale); From aff79eb358f228e463aff7daedec5ef452a23147 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 24 Aug 2020 18:57:44 +0300 Subject: [PATCH 14/26] Prevent default action --- cvat-canvas/src/typescript/interactionHandler.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts index 89f1642742af..8a67546d2b3a 100644 --- a/cvat-canvas/src/typescript/interactionHandler.ts +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -68,6 +68,7 @@ export class InteractionHandlerImpl implements InteractionHandler { private interactPoints(): void { const eventListener = (e: MouseEvent): void => { if ((e.button === 0 || e.button === 2) && !e.altKey) { + e.preventDefault(); const [cx, cy] = translateToSVG( this.canvas.node as any as SVGSVGElement, [e.clientX, e.clientY], From 6ded57b488c2433b874f2b7800e67115a2c133f3 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 25 Aug 2020 18:37:25 +0300 Subject: [PATCH 15/26] Added minNegVertices=0 by default, ignored negative points for dextr, fixed context menu in some cases --- cvat-canvas/src/typescript/canvasModel.ts | 8 ++++++-- .../src/typescript/interactionHandler.ts | 19 +++++++++++++++---- cvat-ui/src/actions/annotation-actions.ts | 2 +- .../controls-side-bar/tools-control.tsx | 15 +++++++++++++-- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts index 8f171483660d..74d9f49cea8f 100644 --- a/cvat-canvas/src/typescript/canvasModel.ts +++ b/cvat-canvas/src/typescript/canvasModel.ts @@ -73,7 +73,8 @@ export interface InteractionData { enabled: boolean; shapeType?: string; crosshair?: boolean; - minVertices?: number; + minPosVertices?: number; + minNegVertices?: number; } export interface InteractionResult { @@ -525,7 +526,10 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel { } this.data.interactionData = interactionData; - this.data.interactionData.crosshair = true; + if (typeof (this.data.interactionData.crosshair) !== 'boolean') { + this.data.interactionData.crosshair = true; + } + this.notify(UpdateReasons.INTERACT); } diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts index 8a67546d2b3a..c8c17c77c573 100644 --- a/cvat-canvas/src/typescript/interactionHandler.ts +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -33,7 +33,7 @@ export class InteractionHandlerImpl implements InteractionHandler { return { points: points.map((coord: number): number => coord - this.geometry.offset), shapeType: 'points', - button: (shape.style('stroke') as any as string) === 'red' ? 2 : 0, + button: shape.attr('stroke') === 'green' ? 0 : 2, }; } @@ -49,10 +49,20 @@ export class InteractionHandlerImpl implements InteractionHandler { private shouldRaiseEvent(ctrlKey: boolean): boolean { const { interactionData, interactionShapes, shapesWereUpdated } = this; - const { minVertices, enabled } = interactionData; + const { minPosVertices, minNegVertices, enabled } = interactionData; - const minimumVerticesAchieved = typeof (minVertices) === 'undefined' - || minVertices <= interactionShapes.length; + const positiveShapes = interactionShapes + .filter((shape: SVG.Shape): boolean => (shape as any).attr('stroke') === 'green'); + const negativeShapes = interactionShapes + .filter((shape: SVG.Shape): boolean => (shape as any).attr('stroke') !== 'green'); + + if (interactionData.shapeType === 'rectangle') { + return enabled && !ctrlKey && !!interactionShapes.length; + } + + const minimumVerticesAchieved = (typeof (minPosVertices) === 'undefined' + || minPosVertices <= positiveShapes.length) && (typeof (minNegVertices) === 'undefined' + || minPosVertices <= negativeShapes.length); return enabled && !ctrlKey && minimumVerticesAchieved && shapesWereUpdated; } @@ -95,6 +105,7 @@ export class InteractionHandlerImpl implements InteractionHandler { }); self.on('mousedown', (_e: MouseEvent): void => { + _e.preventDefault(); _e.stopPropagation(); self.remove(); this.interactionShapes = this.interactionShapes.filter( diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index 64c6a26f310d..efeaa81e8ff0 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -1427,7 +1427,7 @@ export function repeatDrawShapeAsync(): ThunkAction { canvasInstance.interact({ enabled: true, shapeType: 'points', - minVertices: 4, // TODO: Add parameter to interactor + minPosVertices: 4, // TODO: Add parameter to interactor }); dispatch(interactWithCanvas(activeInteractor, activeLabelID)); return; diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 74c365bf0ff9..4e8d53f7e815 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -90,7 +90,7 @@ function convertShapesForInteractor(shapes: InteractionResult[]): number[][] { return acc; }; - return shapes.filter((shape: InteractionResult): boolean => shape.shapeType === 'points') + return shapes.filter((shape: InteractionResult): boolean => shape.shapeType === 'points' && shape.button === 0) .map((shape: InteractionResult): number[] => shape.points) .flat().reduce(reducer, []); } @@ -123,10 +123,13 @@ class ToolsControlComponent extends React.PureComponent { const { isInteraction, jobInstance } = this.props; const { interactiveStateID } = this.state; if (prevProps.isInteraction && !isInteraction) { + window.removeEventListener('contextmenu', this.contextmenuDisabler); if (interactiveStateID !== null) { jobInstance.actions.freeze(false); this.setState({ interactiveStateID: null }); } + } else if (!prevProps.isInteraction && isInteraction) { + window.addEventListener('contextmenu', this.contextmenuDisabler); } } @@ -135,6 +138,13 @@ class ToolsControlComponent extends React.PureComponent { canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener); } + private contextmenuDisabler = (e: MouseEvent): void => { + if (e.target && (e.target as Element).classList + && (e.target as Element).classList.toString().includes('ant-modal')) { + e.preventDefault(); + } + }; + private interactionListener = async (e: Event): Promise => { const { frame, @@ -290,7 +300,7 @@ class ToolsControlComponent extends React.PureComponent { canvasInstance.cancel(); canvasInstance.interact({ shapeType: 'points', - minVertices: 4, // TODO: Add parameter to interactor + minPosVertices: 4, // TODO: Add parameter to interactor enabled: true, }); @@ -349,6 +359,7 @@ class ToolsControlComponent extends React.PureComponent { visible={fetching} closable={false} footer={[]} + > Waiting for a server response.. From ab54158ec16b4d76c9a4fffe94fcc91de1762631 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 27 Aug 2020 17:58:57 +0300 Subject: [PATCH 16/26] On the fly annotations draft --- .../controls-side-bar/tools-control.tsx | 109 +++- .../standard-workspace/styles.scss | 2 +- .../model-runner-modal/detector-runner.tsx | 322 ++++++++++++ .../model-runner-modal/model-runner-modal.tsx | 496 +++--------------- .../components/model-runner-modal/styles.scss | 4 +- .../src/components/task-page/task-page.tsx | 4 +- .../src/components/tasks-page/task-list.tsx | 4 +- .../model-runner-dialog.tsx | 2 +- cvat-ui/src/styles.scss | 4 + 9 files changed, 490 insertions(+), 457 deletions(-) create mode 100644 cvat-ui/src/components/model-runner-modal/detector-runner.tsx diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 4e8d53f7e815..be7cbee55610 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -10,6 +10,7 @@ import Select, { OptionProps } from 'antd/lib/select'; import Button from 'antd/lib/button'; import Modal from 'antd/lib/modal'; import Text from 'antd/lib/typography/Text'; +import Tabs from 'antd/lib/tabs'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; @@ -23,8 +24,14 @@ import { ObjectType, ShapeType, } from 'reducers/interfaces'; -import { interactWithCanvas, fetchAnnotationsAsync, updateAnnotationsAsync } from 'actions/annotation-actions'; +import { + interactWithCanvas, + fetchAnnotationsAsync, + updateAnnotationsAsync, + createAnnotationsAsync, +} from 'actions/annotation-actions'; import { InteractionResult } from 'cvat-canvas/src/typescript/canvas'; +import DetectorRunner from 'components/model-runner-modal/detector-runner'; interface StateToProps { canvasInstance: Canvas; @@ -35,11 +42,14 @@ interface StateToProps { isInteraction: boolean; frame: number; interactors: Model[]; + detectors: Model[]; + reid: Model[]; } interface DispatchToProps { onInteractionStart(activeInteractor: Model, activeLabelID: number): void; updateAnnotations(statesToUpdate: any[]): void; + createAnnotations(sessionInstance: any, frame: number, statesToCreate: any[]): void; fetchAnnotations(): void; } @@ -51,10 +61,12 @@ function mapStateToProps(state: CombinedState): StateToProps { const { instance: jobInstance } = annotation.job; const { instance: canvasInstance, activeControl } = annotation.canvas; const { models } = state; - const { interactors } = models; + const { interactors, detectors, reid } = models; return { interactors, + detectors, + reid, isInteraction: activeControl === ActiveControl.INTERACTION, activeLabelID: annotation.drawing.activeLabelID, labels: annotation.job.labels, @@ -65,19 +77,12 @@ function mapStateToProps(state: CombinedState): StateToProps { }; } -function mapDispatchToProps(dispatch: any): DispatchToProps { - return { - onInteractionStart(activeInteractor: Model, activeLabelID: number): void { - dispatch(interactWithCanvas(activeInteractor, activeLabelID)); - }, - updateAnnotations(statesToUpdate: any[]): void { - dispatch(updateAnnotationsAsync(statesToUpdate)); - }, - fetchAnnotations(): void { - dispatch(fetchAnnotationsAsync()); - }, - }; -} +const mapDispatchToProps = { + onInteractionStart: interactWithCanvas, + updateAnnotations: updateAnnotationsAsync, + fetchAnnotations: fetchAnnotationsAsync, + createAnnotations: createAnnotationsAsync, +}; function convertShapesForInteractor(shapes: InteractionResult[]): number[][] { const reducer = (acc: number[][], _: number, index: number, array: number[]): number[][] => { @@ -289,9 +294,10 @@ class ToolsControlComponent extends React.PureComponent { - - + + + + +
+ ); +} + + +export default React.memo(DetectorRunner, (prevProps: Props, nextProps: Props): boolean => ( + prevProps.task === nextProps.task + && prevProps.runInference === nextProps.runInference + && prevProps.models.length === nextProps.models.length + && nextProps.models.reduce((acc: boolean, model: Model, index: number): boolean => ( + acc && model.id === prevProps.models[index].id + ), true) +)); diff --git a/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx b/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx index d8fd959ecb96..e26eee0809cd 100644 --- a/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx +++ b/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx @@ -4,447 +4,85 @@ import './styles.scss'; import React from 'react'; -import { Row, Col } from 'antd/lib/grid'; -import Icon from 'antd/lib/icon'; -import Select from 'antd/lib/select'; -import Checkbox from 'antd/lib/checkbox'; -import Tooltip from 'antd/lib/tooltip'; +import { connect } from 'react-redux'; import Modal from 'antd/lib/modal'; -import Tag from 'antd/lib/tag'; -import notification from 'antd/lib/notification'; -import Text from 'antd/lib/typography/Text'; -import InputNumber from 'antd/lib/input-number'; -import { - Model, - StringObject, -} from 'reducers/interfaces'; +import { ThunkDispatch } from 'utils/redux'; +import { modelsActions, startInferenceAsync } from 'actions/models-actions'; +import { Model, CombinedState } from 'reducers/interfaces'; +import DetectorRunner from './detector-runner'; -interface Props { - reid: Model[]; - detectors: Model[]; - activeProcesses: StringObject; + +interface StateToProps { visible: boolean; - taskInstance: any; - closeDialog(): void; - runInference( - taskInstance: any, - model: Model, - body: object, - ): void; + task: any; + detectors: Model[]; + reid: Model[]; } -interface State { - selectedModel: string | null; - cleanup: boolean; - mapping: StringObject; - colors: StringObject; - matching: { - model: string; - task: string; - }; - - threshold: number; - maxDistance: number; +interface DispatchToProps { + runInference(task: any, model: Model, body: object): void; + closeDialog(): void; } -function colorGenerator(): () => string { - const values = [ - 'magenta', 'green', 'geekblue', - 'orange', 'red', 'cyan', - 'blue', 'volcano', 'purple', - ]; +function mapStateToProps(state: CombinedState): StateToProps { + const { models } = state; + const { detectors, reid } = models; - let index = 0; - - return (): string => { - const color = values[index++]; - if (index >= values.length) { - index = 0; - } - - return color; + return { + visible: models.visibleRunWindows, + task: models.activeRunTask, + reid, + detectors, }; } -const nextColor = colorGenerator(); - -export default class ModelRunnerModalComponent extends React.PureComponent { - public constructor(props: Props) { - super(props); - this.state = { - selectedModel: null, - mapping: {}, - colors: {}, - cleanup: false, - matching: { - model: '', - task: '', - }, - - threshold: 0.5, - maxDistance: 50, - }; - } - - public componentDidUpdate(prevProps: Props, prevState: State): void { - const { - reid, - detectors, - taskInstance, - visible, - } = this.props; - - const { selectedModel } = this.state; - const models = [...reid, ...detectors]; - - if (!prevProps.visible && visible) { - this.setState({ - selectedModel: null, - mapping: {}, - matching: { - model: '', - task: '', - }, - cleanup: false, - }); - } - - if (selectedModel && prevState.selectedModel !== selectedModel) { - const selectedModelInstance = models - .filter((model) => model.name === selectedModel)[0]; - - if (selectedModelInstance.type !== 'reid' && !selectedModelInstance.labels.length) { - notification.warning({ - message: 'The selected model does not include any lables', - }); - } - - let taskLabels: string[] = taskInstance.labels - .map((label: any): string => label.name); - const [defaultMapping, defaultColors]: StringObject[] = selectedModelInstance.labels - .reduce((acc: StringObject[], label): StringObject[] => { - if (taskLabels.includes(label)) { - acc[0][label] = label; - acc[1][label] = nextColor(); - taskLabels = taskLabels.filter((_label): boolean => _label !== label); - } - - return acc; - }, [{}, {}]); - - this.setState({ - mapping: defaultMapping, - colors: defaultColors, - }); - } - } - - private renderModelSelector(): JSX.Element { - const { reid, detectors } = this.props; - const models = [...reid, ...detectors]; - - return ( - - Model: - - - - - ); - } - - private renderMappingTag(modelLabel: string, taskLabel: string): JSX.Element { - const { colors, mapping } = this.state; - - return ( - - - {modelLabel} - - - {taskLabel} - - - - { - const newMapping = { ...mapping }; - delete newMapping[modelLabel]; - this.setState({ - mapping: newMapping, - }); - }} - /> - - - - ); - } - - private renderMappingInputSelector( - value: string, - current: string, - options: string[], - ): JSX.Element { - const { matching, mapping, colors } = this.state; - - return ( - - ); - } - - private renderMappingInput( - availableModelLabels: string[], - availableTaskLabels: string[], - ): JSX.Element { - const { matching } = this.state; - return ( - - - {this.renderMappingInputSelector( - matching.model, - 'Model', - availableModelLabels, - )} - - - {this.renderMappingInputSelector( - matching.task, - 'Task', - availableTaskLabels, - )} - - - - - - - - ); - } - - private renderReidContent(): JSX.Element { - const { threshold, maxDistance } = this.state; - - return ( -
- - - Threshold - - - - { - if (typeof (value) === 'number') { - this.setState({ - threshold: value, - }); - } - }} - /> - - - - - - Maximum distance - - - - { - if (typeof (value) === 'number') { - this.setState({ - maxDistance: value, - }); - } - }} - /> - - - -
- ); - } - - private renderContent(): JSX.Element { - const { selectedModel, cleanup, mapping } = this.state; - const { reid, detectors, taskInstance } = this.props; - - const models = [...reid, ...detectors]; - const model = selectedModel && models - .filter((_model): boolean => _model.name === selectedModel)[0]; - - const excludedModelLabels: string[] = Object.keys(mapping); - const isDetector = model && model.type === 'detector'; - const isReId = model && model.type === 'reid'; - const tags = isDetector ? excludedModelLabels - .map((modelLabel: string) => this.renderMappingTag( - modelLabel, - mapping[modelLabel], - )) : []; - - const availableModelLabels = model ? model.labels - .filter( - (label: string) => !excludedModelLabels.includes(label), - ) : []; - const taskLabels = taskInstance.labels.map( - (label: any) => label.name, - ); - - const mappingISAvailable = !!availableModelLabels.length - && !!taskLabels.length; - - return ( -
- { this.renderModelSelector() } - { isDetector && tags} - { isDetector - && mappingISAvailable - && this.renderMappingInput(availableModelLabels, taskLabels)} - { isDetector - && ( -
- this.setState({ - cleanup: e.target.checked, - })} - > - Clean old annotations - -
- )} - { isReId && this.renderReidContent() } -
- ); - } - - public render(): JSX.Element | false { - const { - selectedModel, - mapping, - cleanup, - threshold, - maxDistance, - } = this.state; - - const { - reid, - detectors, - visible, - taskInstance, - runInference, - closeDialog, - } = this.props; - - const models = [...reid, ...detectors]; - const activeModel = models.filter( - (model): boolean => model.name === selectedModel, - )[0]; - - const enabledSubmit = !!activeModel && (activeModel.type === 'reid' - || !!Object.keys(mapping).length); - - return ( - visible && ( - { - runInference( - taskInstance, - models - .filter((model): boolean => model.name === selectedModel)[0], - activeModel.type === 'detector' ? { - mapping, - cleanup, - } : { - threshold, - max_distance: maxDistance, - }, - ); - closeDialog(); - }} - onCancel={(): void => closeDialog()} - okButtonProps={{ disabled: !enabledSubmit }} - title='Automatic annotation' - visible - > - { this.renderContent() } - - ) - ); - } + /> + + ); } + +export default connect( + mapStateToProps, + mapDispatchToProps, +)(ModelRunnerModal); diff --git a/cvat-ui/src/components/model-runner-modal/styles.scss b/cvat-ui/src/components/model-runner-modal/styles.scss index 9c9bc0c0deca..80bef525d559 100644 --- a/cvat-ui/src/components/model-runner-modal/styles.scss +++ b/cvat-ui/src/components/model-runner-modal/styles.scss @@ -4,10 +4,10 @@ @import '../../base.scss'; -.cvat-run-model-dialog > div:not(first-child) { +.cvat-run-model-content > div:not(first-child) { margin-top: 10px; } -.cvat-run-model-dialog-remove-mapping-icon { +.cvat-run-model-content-remove-mapping-icon { color: $danger-icon-color; } diff --git a/cvat-ui/src/components/task-page/task-page.tsx b/cvat-ui/src/components/task-page/task-page.tsx index c66db67164b8..9ea5006c5772 100644 --- a/cvat-ui/src/components/task-page/task-page.tsx +++ b/cvat-ui/src/components/task-page/task-page.tsx @@ -12,7 +12,7 @@ import Result from 'antd/lib/result'; import DetailsContainer from 'containers/task-page/details'; import JobListContainer from 'containers/task-page/job-list'; -import ModelRunnerModalContainer from 'containers/model-runner-dialog/model-runner-dialog'; +import ModelRunnerModal from 'components/model-runner-modal/model-runner-modal'; import { Task } from 'reducers/interfaces'; import TopBarComponent from './top-bar'; @@ -75,7 +75,7 @@ class TaskPageComponent extends React.PureComponent { - + ); } diff --git a/cvat-ui/src/components/tasks-page/task-list.tsx b/cvat-ui/src/components/tasks-page/task-list.tsx index 68515a628f22..9710d439fb5f 100644 --- a/cvat-ui/src/components/tasks-page/task-list.tsx +++ b/cvat-ui/src/components/tasks-page/task-list.tsx @@ -6,7 +6,7 @@ import React from 'react'; import { Row, Col } from 'antd/lib/grid'; import Pagination from 'antd/lib/pagination'; -import ModelRunnerModalContainer from 'containers/model-runner-dialog/model-runner-dialog'; +import ModelRunnerModal from 'components/model-runner-modal/model-runner-modal'; import TaskItem from 'containers/tasks-page/task-item'; export interface ContentListProps { @@ -46,7 +46,7 @@ export default function TaskListComponent(props: ContentListProps): JSX.Element /> - + ); } diff --git a/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx b/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx index 3f63bd7f6a11..b9833a56c6d6 100644 --- a/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx +++ b/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx @@ -4,7 +4,7 @@ import { connect } from 'react-redux'; -import ModelRunnerModalComponent from 'components/model-runner-modal/model-runner-modal'; +import ModelRunnerModalComponent from 'components/model-runner-modal/detector-runner'; import { Model, CombinedState } from 'reducers/interfaces'; import { startInferenceAsync, modelsActions } from 'actions/models-actions'; diff --git a/cvat-ui/src/styles.scss b/cvat-ui/src/styles.scss index cbd6a4c2e763..284b2772d46d 100644 --- a/cvat-ui/src/styles.scss +++ b/cvat-ui/src/styles.scss @@ -43,6 +43,10 @@ hr { color: $info-icon-color; } +.cvat-danger-circle-icon { + color: $danger-icon-color; +} + #root { width: 100%; height: 100%; From 25df34be094cb9dfc14f6bdd9c1104f14f8b2417 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 27 Aug 2020 20:21:27 +0300 Subject: [PATCH 17/26] Initial version of FBRS interactive segmentation --- .../saic-vul/fbrs/nuclio/function.yaml | 59 ++++++++++++ .../pytorch/saic-vul/fbrs/nuclio/main.py | 29 ++++++ .../saic-vul/fbrs/nuclio/model_handler.py | 89 +++++++++++++++++++ 3 files changed, 177 insertions(+) create mode 100644 serverless/pytorch/saic-vul/fbrs/nuclio/function.yaml create mode 100644 serverless/pytorch/saic-vul/fbrs/nuclio/main.py create mode 100644 serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py diff --git a/serverless/pytorch/saic-vul/fbrs/nuclio/function.yaml b/serverless/pytorch/saic-vul/fbrs/nuclio/function.yaml new file mode 100644 index 000000000000..ba8156098d7a --- /dev/null +++ b/serverless/pytorch/saic-vul/fbrs/nuclio/function.yaml @@ -0,0 +1,59 @@ +metadata: + name: pth.saic-vul.fbrs + namespace: cvat + annotations: + name: FBRS + type: interactor + spec: + framework: pytorch + min_pos_points: 1 + +spec: + description: f-BRS Rethinking Backpropagating Refinement for Interactive Segmentation + runtime: "python:3.6" + handler: main:handler + eventTimeout: 30s + env: + - name: PYTHONPATH + value: /opt/nuclio/fbrs + + build: + image: cvat/pth.saic-vul.fbrs + baseImage: python:3.6.11 + + directives: + preCopy: + - kind: WORKDIR + value: /opt/nuclio + - kind: RUN + value: git clone https://github.com/saic-vul/fbrs_interactive_segmentation.git fbrs + - kind: WORKDIR + value: /opt/nuclio/fbrs + - kind: ENV + value: fileid=1Z9dQtpWVTobEdmUBntpUU0pJl-pEXUwR + - kind: ENV + value: filename=model.pth + - kind: RUN + value: curl -c ./cookie -s -L "https://drive.google.com/uc?export=download&id=${fileid}" + - kind: RUN + value: curl -Lb ./cookie "https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ./cookie`&id=${fileid}" -o ${filename} + - kind: RUN + value: apt update && apt install -y libgl1-mesa-glx + - kind: RUN + value: pip3 install -r requirements.txt + - kind: WORKDIR + value: /opt/nuclio + + triggers: + myHttpTrigger: + maxWorkers: 2 + kind: "http" + workerAvailabilityTimeoutMilliseconds: 10000 + attributes: + maxRequestBodySize: 33554432 # 32MB + + platform: + attributes: + restartPolicy: + name: always + maximumRetryCount: 3 diff --git a/serverless/pytorch/saic-vul/fbrs/nuclio/main.py b/serverless/pytorch/saic-vul/fbrs/nuclio/main.py new file mode 100644 index 000000000000..1edacfa373db --- /dev/null +++ b/serverless/pytorch/saic-vul/fbrs/nuclio/main.py @@ -0,0 +1,29 @@ +import json +import base64 +from PIL import Image +import io +from model_handler import ModelHandler + +def init_context(context): + context.logger.info("Init context... 0%") + + model = ModelHandler() + setattr(context.user_data, 'model', model) + + context.logger.info("Init context...100%") + +def handler(context, event): + context.logger.info("call handler") + data = event.body + pos_points = data["points"] + neg_points = [] + threshold = data.get("threshold", 0.5) + buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) + image = Image.open(buf) + + polygon = context.user_data.model.handle(image, pos_points, + neg_points, threshold) + return context.Response(body=json.dumps(polygon), + headers={}, + content_type='application/json', + status_code=200) diff --git a/serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py b/serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py new file mode 100644 index 000000000000..8e34e3037d7a --- /dev/null +++ b/serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py @@ -0,0 +1,89 @@ +import torch +import numpy as np +from torchvision import transforms +import cv2 +import os + +from isegm.inference import clicker +from isegm.inference.predictors import get_predictor +from isegm.inference.utils import load_deeplab_is_model, load_hrnet_is_model +from isegm.inference.clicker import Clicker, Click + +def convert_mask_to_polygon(mask): + mask = np.array(mask, dtype=np.uint8) + cv2.normalize(mask, mask, 0, 255, cv2.NORM_MINMAX) + contours = None + if int(cv2.__version__.split('.')[0]) > 3: + contours = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[0] + else: + contours = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[1] + + contours = max(contours, key=lambda arr: arr.size) + if contours.shape.count(1): + contours = np.squeeze(contours) + if contours.size < 3 * 2: + raise Exception('Less then three point have been detected. Can not build a polygon.') + + polygon = [] + for point in contours: + polygon.append([int(point[0]), int(point[1])]) + + return polygon + +class ModelHandler: + def __init__(self): + torch.backends.cudnn.deterministic = True + base_dir = os.environ.get("MODEL_PATH", "/opt/nuclio") + model_path = os.path.join(base_dir, "resnet101_dh256_sbd.pth") + state_dict = torch.load(model_path, map_location='cpu') + + self.net = None + backbone = 'auto' + for k in state_dict.keys(): + if 'feature_extractor.stage2.0.branches' in k: + self.net = load_hrnet_is_model(state_dict, device, backbone, **kwargs) + break + + self.device = 'cuda' if torch.cuda.is_available() else 'cpu' + if self.net is None: + self.net = load_deeplab_is_model(state_dict, self.device, backbone) + self.net.to(self.device) + + def handle(self, image, pos_points, neg_points, threshold): + input_transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize([.485, .456, .406], [.229, .224, .225]) + ]) + + image_nd = input_transform(image).to(self.device) + result_mask = np.zeros(image.shape[:2], dtype=np.uint16) + + clicker = Clicker() + for x, y in pos_points: + click = Click(is_positive=True, coords=(y, x)) + clicker.add_click(click) + + for x, y in neg_points: + click = Click(is_positive=False, coords=(y, x)) + clicker.add_click(click) + + predictor_params = { + 'brs_mode': 'f-BRS-B', + 'brs_opt_func_params': {'min_iou_diff': 0.001}, + 'lbfgs_params': {'maxfun': 20}, + 'predictor_params': {'max_size': 800, 'net_clicks_limit': 8}, + 'prob_thresh': threshold, + 'zoom_in_params': {'expansion_ratio': 1.4, 'skip_clicks': 1, 'target_size': 480}} + predictor = get_predictor(self.net, device=self.device, + **predictor_params) + predictor.set_input_image(image_nd) + + object_prob = predictor.get_prediction(clicker) + if self.device == 'cuda': + torch.cuda.empty_cache() + object_mask = object_prob > threshold + polygon = convert_mask_to_polygon(object_mask) + + return polygon + + From 820e4989c4fa7e57dded2494a7ac26213d3699b9 Mon Sep 17 00:00:00 2001 From: Nikita Manovich Date: Thu, 27 Aug 2020 20:39:52 +0300 Subject: [PATCH 18/26] Fix fbrs model_handler --- serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py b/serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py index 8e34e3037d7a..e4d86cdf4b58 100644 --- a/serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py +++ b/serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py @@ -33,8 +33,8 @@ def convert_mask_to_polygon(mask): class ModelHandler: def __init__(self): torch.backends.cudnn.deterministic = True - base_dir = os.environ.get("MODEL_PATH", "/opt/nuclio") - model_path = os.path.join(base_dir, "resnet101_dh256_sbd.pth") + base_dir = os.environ.get("MODEL_PATH", "/opt/nuclio/fbrs") + model_path = os.path.join(base_dir, "model.pth") state_dict = torch.load(model_path, map_location='cpu') self.net = None @@ -56,7 +56,7 @@ def handle(self, image, pos_points, neg_points, threshold): ]) image_nd = input_transform(image).to(self.device) - result_mask = np.zeros(image.shape[:2], dtype=np.uint16) + result_mask = np.zeros(image_nd.shape[:2], dtype=np.uint16) clicker = Clicker() for x, y in pos_points: From daeaf75b379dcdd506fcb6a26b8218b4e98640c6 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 28 Aug 2020 12:36:28 +0300 Subject: [PATCH 19/26] Fixed couple of minor bugs --- cvat-canvas/src/typescript/canvasView.ts | 35 ++++--- .../src/typescript/interactionHandler.ts | 39 ++++---- .../controls-side-bar/tools-control.tsx | 97 ++++++++++++++----- 3 files changed, 114 insertions(+), 57 deletions(-) diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts index 13dc922e02c1..3cf37f045a85 100644 --- a/cvat-canvas/src/typescript/canvasView.ts +++ b/cvat-canvas/src/typescript/canvasView.ts @@ -131,26 +131,19 @@ export class CanvasViewImpl implements CanvasView, Listener { } } - private onStopInteraction(): void { - const event: CustomEvent = new CustomEvent('canvas.canceled', { - bubbles: false, - cancelable: true, - }); - - this.canvas.dispatchEvent(event); - this.mode = Mode.IDLE; - this.controller.interact({ - enabled: false, - }); - } - - private onInteraction(shapes: InteractionResult[] | null): void { + private onInteraction( + shapes: InteractionResult[] | null, + shapesUpdated: boolean = true, + isDone: boolean = false, + ): void { const { zLayer } = this.controller; if (Array.isArray(shapes)) { const event: CustomEvent = new CustomEvent('canvas.interacted', { bubbles: false, cancelable: true, detail: { + shapesUpdated, + isDone, shapes, zOrder: zLayer || 0, }, @@ -158,6 +151,19 @@ export class CanvasViewImpl implements CanvasView, Listener { this.canvas.dispatchEvent(event); } + + if (shapes === null || isDone) { + const event: CustomEvent = new CustomEvent('canvas.canceled', { + bubbles: false, + cancelable: true, + }); + + this.canvas.dispatchEvent(event); + this.mode = Mode.IDLE; + this.controller.interact({ + enabled: false, + }); + } } private onDrawDone(data: object | null, duration: number, continueDraw?: boolean): void { @@ -885,7 +891,6 @@ export class CanvasViewImpl implements CanvasView, Listener { ); this.interactionHandler = new InteractionHandlerImpl( this.onInteraction.bind(this), - this.onStopInteraction.bind(this), this.adoptedContent, this.geometry, ); diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts index c8c17c77c573..76237cf06ed4 100644 --- a/cvat-canvas/src/typescript/interactionHandler.ts +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -15,8 +15,11 @@ export interface InteractionHandler { } export class InteractionHandlerImpl implements InteractionHandler { - private onInteraction: (shapes: InteractionResult[] | null) => void; - private onStopInteraction: () => void; + private onInteraction: ( + shapes: InteractionResult[] | null, + shapesUpdated?: boolean, + isDone?: boolean, + ) => void; private geometry: Geometry; private canvas: SVG.Container; private interactionData: InteractionData; @@ -95,7 +98,7 @@ export class InteractionHandlerImpl implements InteractionHandler { this.interactionShapes.push(this.currentInteractionShape); this.shapesWereUpdated = true; if (this.shouldRaiseEvent(e.ctrlKey)) { - this.onInteraction(this.prepareResult()); + this.onInteraction(this.prepareResult(), true, false); } const self = this.currentInteractionShape; @@ -113,7 +116,7 @@ export class InteractionHandlerImpl implements InteractionHandler { ); this.shapesWereUpdated = true; if (this.shouldRaiseEvent(_e.ctrlKey)) { - this.onInteraction(this.prepareResult()); + this.onInteraction(this.prepareResult(), true, false); } }); }); @@ -153,7 +156,7 @@ export class InteractionHandlerImpl implements InteractionHandler { this.canvas.off('mousedown.interaction', eventListener); if (this.shouldRaiseEvent(false)) { - this.onInteraction(this.prepareResult()); + this.onInteraction(this.prepareResult(), true, false); } this.interact({ enabled: false }); @@ -190,21 +193,25 @@ export class InteractionHandlerImpl implements InteractionHandler { this.currentInteractionShape.remove(); this.currentInteractionShape = null; } - - this.onStopInteraction(); } public constructor( - onInteraction: (shapes: InteractionResult[] | null) => void, - onStopInteraction: () => void, + onInteraction: ( + shapes: InteractionResult[] | null, + shapesUpdated?: boolean, + isDone?: boolean, + ) => void, canvas: SVG.Container, geometry: Geometry, ) { - this.onInteraction = (shapes: InteractionResult[] | null): void => { + this.onInteraction = ( + shapes: InteractionResult[] | null, + shapesUpdated?: boolean, + isDone?: boolean, + ): void => { this.shapesWereUpdated = false; - onInteraction(shapes); + onInteraction(shapes, shapesUpdated, isDone); }; - this.onStopInteraction = onStopInteraction; this.canvas = canvas; this.geometry = geometry; this.shapesWereUpdated = false; @@ -230,7 +237,7 @@ export class InteractionHandlerImpl implements InteractionHandler { document.body.addEventListener('keyup', (e: KeyboardEvent): void => { if (e.keyCode === 17 && this.shouldRaiseEvent(false)) { // 17 is ctrl - this.onInteraction(this.prepareResult()); + this.onInteraction(this.prepareResult(), true, false); } }); } @@ -261,10 +268,7 @@ export class InteractionHandlerImpl implements InteractionHandler { this.initInteraction(); this.startInteraction(); } else { - if (this.shouldRaiseEvent(false)) { - this.onInteraction(this.prepareResult()); - } - + this.onInteraction(this.prepareResult(), this.shouldRaiseEvent(false), true); this.release(); this.interactionData = interactionData; } @@ -272,5 +276,6 @@ export class InteractionHandlerImpl implements InteractionHandler { public cancel(): void { this.release(); + this.onInteraction(null); } } diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 4e8d53f7e815..c661574877d1 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -117,17 +117,13 @@ class ToolsControlComponent extends React.PureComponent { public componentDidMount(): void { const { canvasInstance } = this.props; canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener); + canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener); } public componentDidUpdate(prevProps: Props): void { - const { isInteraction, jobInstance } = this.props; - const { interactiveStateID } = this.state; + const { isInteraction } = this.props; if (prevProps.isInteraction && !isInteraction) { window.removeEventListener('contextmenu', this.contextmenuDisabler); - if (interactiveStateID !== null) { - jobInstance.actions.freeze(false); - this.setState({ interactiveStateID: null }); - } } else if (!prevProps.isInteraction && isInteraction) { window.addEventListener('contextmenu', this.contextmenuDisabler); } @@ -136,6 +132,14 @@ class ToolsControlComponent extends React.PureComponent { public componentWillUnmount(): void { const { canvasInstance } = this.props; canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener); + canvasInstance.html().removeEventListener('canvas.canceled', this.cancelListener); + } + + private getInteractiveState(): any | null { + const { states } = this.props; + const { interactiveStateID } = this.state; + return states + .filter((_state: any): boolean => _state.clientID === interactiveStateID)[0] || null; } private contextmenuDisabler = (e: MouseEvent): void => { @@ -145,10 +149,30 @@ class ToolsControlComponent extends React.PureComponent { } }; + private cancelListener = async (): Promise => { + const { + isInteraction, + jobInstance, + frame, + fetchAnnotations, + } = this.props; + const { interactiveStateID } = this.state; + + if (isInteraction) { + if (interactiveStateID !== null) { + const state = this.getInteractiveState(); + this.setState({ interactiveStateID: null }); + await state.delete(frame); + fetchAnnotations(); + } + + await jobInstance.actions.freeze(false); + } + }; + private interactionListener = async (e: Event): Promise => { const { frame, - states, labels, jobInstance, isInteraction, @@ -159,21 +183,31 @@ class ToolsControlComponent extends React.PureComponent { const { activeInteractor, interactiveStateID } = this.state; try { - this.setState({ fetching: true }); - if (!isInteraction) { - throw Error('Canvas raises "canvas.interacted" when interaction is off'); + throw Error('Canvas raises event "canvas.interacted" when interaction is off'); } const interactor = activeInteractor as Model; - const result = await core.lambda.call(jobInstance.task, interactor, { - task: jobInstance.task, - frame, - points: convertShapesForInteractor((e as CustomEvent).detail.shapes), - }); + + let result = []; + if ((e as CustomEvent).detail.shapesUpdated) { + this.setState({ fetching: true }); + try { + result = await core.lambda.call(jobInstance.task, interactor, { + task: jobInstance.task, + frame, + points: convertShapesForInteractor((e as CustomEvent).detail.shapes), + }); + } finally { + this.setState({ fetching: false }); + } + } // no shape yet, then create it and save to collection if (interactiveStateID === null) { + // freeze history for interaction time + // (points updating shouldn't cause adding new actions to history) + await jobInstance.actions.freeze(true); const object = new core.classes.ObjectState({ frame, objectType: ObjectType.SHAPE, @@ -184,29 +218,42 @@ class ToolsControlComponent extends React.PureComponent { occluded: false, zOrder: (e as CustomEvent).detail.zOrder, }); - // need a clientID of a created object, so, we do not use createAnnotationAction + // need a clientID of a created object to interact with it further + // so, we do not use createAnnotationAction const [clientID] = await jobInstance.annotations.put([object]); // update annotations on a canvas fetchAnnotations(); - - // freeze history for interaction time - // (points updating shouldn't cause adding new actions to history) - await jobInstance.actions.freeze(true); this.setState({ interactiveStateID: clientID }); + return; + } + + const state = this.getInteractiveState(); + if ((e as CustomEvent).detail.isDone) { + const finalObject = new core.classes.ObjectState({ + frame: state.frame, + objectType: state.objectType, + label: state.label, + shapeType: state.shapeType, + points: result.length ? result.flat() : state.points, + occluded: state.occluded, + zOrder: state.zOrder, + }); + this.setState({ interactiveStateID: null }); + await state.delete(frame); + await jobInstance.actions.freeze(false); + await jobInstance.annotations.put([finalObject]); + fetchAnnotations(); } else { - const state = states - .filter((_state: any): boolean => _state.clientID === interactiveStateID)[0]; state.points = result.flat(); - await updateAnnotations([state]); + updateAnnotations([state]); + fetchAnnotations(); } } catch (err) { notification.error({ description: err.toString(), message: 'Interaction error occured', }); - } finally { - this.setState({ fetching: false }); } }; From 9e7ebcc937794545f78d32026c23caef06f8eb12 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 28 Aug 2020 13:04:24 +0300 Subject: [PATCH 20/26] Added ability to interrupt interaction --- .../controls-side-bar/tools-control.tsx | 110 ++++++++++++------ 1 file changed, 76 insertions(+), 34 deletions(-) diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index c661574877d1..5bdae3d45ade 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -104,6 +104,9 @@ interface State { } class ToolsControlComponent extends React.PureComponent { + private interactionIsAborted: boolean; + private interactionIsDone: boolean; + public constructor(props: Props) { super(props); this.state = { @@ -112,6 +115,9 @@ class ToolsControlComponent extends React.PureComponent { interactiveStateID: null, fetching: false, }; + + this.interactionIsAborted = false; + this.interactionIsDone = false; } public componentDidMount(): void { @@ -125,6 +131,8 @@ class ToolsControlComponent extends React.PureComponent { if (prevProps.isInteraction && !isInteraction) { window.removeEventListener('contextmenu', this.contextmenuDisabler); } else if (!prevProps.isInteraction && isInteraction) { + this.interactionIsDone = false; + this.interactionIsAborted = false; window.addEventListener('contextmenu', this.contextmenuDisabler); } } @@ -156,9 +164,15 @@ class ToolsControlComponent extends React.PureComponent { frame, fetchAnnotations, } = this.props; - const { interactiveStateID } = this.state; + const { interactiveStateID, fetching } = this.state; if (isInteraction) { + if (fetching && !this.interactionIsDone) { + // user pressed ESC + this.setState({ fetching: false }); + this.interactionIsAborted = true; + } + if (interactiveStateID !== null) { const state = this.getInteractiveState(); this.setState({ interactiveStateID: null }); @@ -180,13 +194,18 @@ class ToolsControlComponent extends React.PureComponent { fetchAnnotations, updateAnnotations, } = this.props; - const { activeInteractor, interactiveStateID } = this.state; + const { activeInteractor, interactiveStateID, fetching } = this.state; try { if (!isInteraction) { throw Error('Canvas raises event "canvas.interacted" when interaction is off'); } + if (fetching) { + this.interactionIsDone = (e as CustomEvent).detail.isDone; + return; + } + const interactor = activeInteractor as Model; let result = []; @@ -198,16 +217,19 @@ class ToolsControlComponent extends React.PureComponent { frame, points: convertShapesForInteractor((e as CustomEvent).detail.shapes), }); + + if (this.interactionIsAborted) { + // while the server request + // user has cancelled interaction (for example pressed ESC) + return; + } } finally { this.setState({ fetching: false }); } } - // no shape yet, then create it and save to collection - if (interactiveStateID === null) { - // freeze history for interaction time - // (points updating shouldn't cause adding new actions to history) - await jobInstance.actions.freeze(true); + if (this.interactionIsDone) { + // while the server request, user has done interaction (for example pressed N) const object = new core.classes.ObjectState({ frame, objectType: ObjectType.SHAPE, @@ -218,36 +240,56 @@ class ToolsControlComponent extends React.PureComponent { occluded: false, zOrder: (e as CustomEvent).detail.zOrder, }); - // need a clientID of a created object to interact with it further - // so, we do not use createAnnotationAction - const [clientID] = await jobInstance.annotations.put([object]); - - // update annotations on a canvas - fetchAnnotations(); - this.setState({ interactiveStateID: clientID }); - return; - } - const state = this.getInteractiveState(); - if ((e as CustomEvent).detail.isDone) { - const finalObject = new core.classes.ObjectState({ - frame: state.frame, - objectType: state.objectType, - label: state.label, - shapeType: state.shapeType, - points: result.length ? result.flat() : state.points, - occluded: state.occluded, - zOrder: state.zOrder, - }); - this.setState({ interactiveStateID: null }); - await state.delete(frame); - await jobInstance.actions.freeze(false); - await jobInstance.annotations.put([finalObject]); + await jobInstance.annotations.put([object]); fetchAnnotations(); } else { - state.points = result.flat(); - updateAnnotations([state]); - fetchAnnotations(); + // no shape yet, then create it and save to collection + if (interactiveStateID === null) { + // freeze history for interaction time + // (points updating shouldn't cause adding new actions to history) + await jobInstance.actions.freeze(true); + const object = new core.classes.ObjectState({ + frame, + objectType: ObjectType.SHAPE, + label: labels + .filter((label: any) => label.id === activeLabelID)[0], + shapeType: ShapeType.POLYGON, + points: result.flat(), + occluded: false, + zOrder: (e as CustomEvent).detail.zOrder, + }); + // need a clientID of a created object to interact with it further + // so, we do not use createAnnotationAction + const [clientID] = await jobInstance.annotations.put([object]); + + // update annotations on a canvas + fetchAnnotations(); + this.setState({ interactiveStateID: clientID }); + return; + } + + const state = this.getInteractiveState(); + if ((e as CustomEvent).detail.isDone) { + const finalObject = new core.classes.ObjectState({ + frame: state.frame, + objectType: state.objectType, + label: state.label, + shapeType: state.shapeType, + points: result.length ? result.flat() : state.points, + occluded: state.occluded, + zOrder: state.zOrder, + }); + this.setState({ interactiveStateID: null }); + await state.delete(frame); + await jobInstance.actions.freeze(false); + await jobInstance.annotations.put([finalObject]); + fetchAnnotations(); + } else { + state.points = result.flat(); + updateAnnotations([state]); + fetchAnnotations(); + } } } catch (err) { notification.error({ From 0b7f89cb40fd769755f3b44ed82961f05a101b3e Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 31 Aug 2020 10:49:45 +0300 Subject: [PATCH 21/26] Do not show reid on annotation view --- .../controls-side-bar/tools-control.tsx | 7 +-- ...nner-modal.tsx => model-runner-dialog.tsx} | 4 +- .../src/components/task-page/task-page.tsx | 2 +- .../src/components/tasks-page/task-list.tsx | 2 +- .../model-runner-dialog.tsx | 60 ------------------- 5 files changed, 6 insertions(+), 69 deletions(-) rename cvat-ui/src/components/model-runner-modal/{model-runner-modal.tsx => model-runner-dialog.tsx} (95%) delete mode 100644 cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 965332e25f9e..096d319ea1c4 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -43,7 +43,6 @@ interface StateToProps { frame: number; interactors: Model[]; detectors: Model[]; - reid: Model[]; } interface DispatchToProps { @@ -61,12 +60,11 @@ function mapStateToProps(state: CombinedState): StateToProps { const { instance: jobInstance } = annotation.job; const { instance: canvasInstance, activeControl } = annotation.canvas; const { models } = state; - const { interactors, detectors, reid } = models; + const { interactors, detectors } = models; return { interactors, detectors, - reid, isInteraction: activeControl === ActiveControl.INTERACTION, activeLabelID: annotation.drawing.activeLabelID, labels: annotation.job.labels, @@ -415,7 +413,6 @@ class ToolsControlComponent extends React.PureComponent { const { jobInstance, detectors, - reid, frame, fetchAnnotations, } = this.props; @@ -423,7 +420,7 @@ class ToolsControlComponent extends React.PureComponent { return ( { try { diff --git a/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx b/cvat-ui/src/components/model-runner-modal/model-runner-dialog.tsx similarity index 95% rename from cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx rename to cvat-ui/src/components/model-runner-modal/model-runner-dialog.tsx index e26eee0809cd..cff9914bf7f1 100644 --- a/cvat-ui/src/components/model-runner-modal/model-runner-modal.tsx +++ b/cvat-ui/src/components/model-runner-modal/model-runner-dialog.tsx @@ -48,7 +48,7 @@ function mapDispatchToProps(dispatch: ThunkDispatch): DispatchToProps { }; } -function ModelRunnerModal(props: StateToProps & DispatchToProps): JSX.Element { +function ModelRunnerDialog(props: StateToProps & DispatchToProps): JSX.Element { const { reid, detectors, @@ -85,4 +85,4 @@ function ModelRunnerModal(props: StateToProps & DispatchToProps): JSX.Element { export default connect( mapStateToProps, mapDispatchToProps, -)(ModelRunnerModal); +)(ModelRunnerDialog); diff --git a/cvat-ui/src/components/task-page/task-page.tsx b/cvat-ui/src/components/task-page/task-page.tsx index 9ea5006c5772..ea6cdf803a1b 100644 --- a/cvat-ui/src/components/task-page/task-page.tsx +++ b/cvat-ui/src/components/task-page/task-page.tsx @@ -12,7 +12,7 @@ import Result from 'antd/lib/result'; import DetailsContainer from 'containers/task-page/details'; import JobListContainer from 'containers/task-page/job-list'; -import ModelRunnerModal from 'components/model-runner-modal/model-runner-modal'; +import ModelRunnerModal from 'components/model-runner-modal/model-runner-dialog'; import { Task } from 'reducers/interfaces'; import TopBarComponent from './top-bar'; diff --git a/cvat-ui/src/components/tasks-page/task-list.tsx b/cvat-ui/src/components/tasks-page/task-list.tsx index 9710d439fb5f..01bfcb6a26ba 100644 --- a/cvat-ui/src/components/tasks-page/task-list.tsx +++ b/cvat-ui/src/components/tasks-page/task-list.tsx @@ -6,7 +6,7 @@ import React from 'react'; import { Row, Col } from 'antd/lib/grid'; import Pagination from 'antd/lib/pagination'; -import ModelRunnerModal from 'components/model-runner-modal/model-runner-modal'; +import ModelRunnerModal from 'components/model-runner-modal/model-runner-dialog'; import TaskItem from 'containers/tasks-page/task-item'; export interface ContentListProps { diff --git a/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx b/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx deleted file mode 100644 index b9833a56c6d6..000000000000 --- a/cvat-ui/src/containers/model-runner-dialog/model-runner-dialog.tsx +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) 2020 Intel Corporation -// -// SPDX-License-Identifier: MIT - -import { connect } from 'react-redux'; - -import ModelRunnerModalComponent from 'components/model-runner-modal/detector-runner'; -import { Model, CombinedState } from 'reducers/interfaces'; -import { startInferenceAsync, modelsActions } from 'actions/models-actions'; - -interface StateToProps { - reid: Model[]; - detectors: Model[]; - activeProcesses: { - [index: string]: string; - }; - taskInstance: any; - visible: boolean; -} - -interface DispatchToProps { - runInference( - taskInstance: any, - model: Model, - body: object, - ): void; - closeDialog(): void; -} - -function mapStateToProps(state: CombinedState): StateToProps { - const { models } = state; - - return { - reid: models.reid, - detectors: models.detectors, - activeProcesses: {}, - taskInstance: models.activeRunTask, - visible: models.visibleRunWindows, - }; -} - -function mapDispatchToProps(dispatch: any): DispatchToProps { - return ({ - runInference( - taskInstance: any, - model: Model, - body: object, - ): void { - dispatch(startInferenceAsync(taskInstance, model, body)); - }, - closeDialog(): void { - dispatch(modelsActions.closeRunModelDialog()); - }, - }); -} - -export default connect( - mapStateToProps, - mapDispatchToProps, -)(ModelRunnerModalComponent); From a15200bf830ff86b5266511c3a1b8a9a1703f136 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 31 Aug 2020 11:08:32 +0300 Subject: [PATCH 22/26] Prettified UI --- .../standard-workspace/controls-side-bar/tools-control.tsx | 2 +- .../src/components/model-runner-modal/detector-runner.tsx | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 096d319ea1c4..78c7832b0fb6 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -466,7 +466,7 @@ class ToolsControlComponent extends React.PureComponent { private renderPopoverContent(): JSX.Element { return ( -
+
false} className='cvat-tools-control-popover-content'> AI Tools diff --git a/cvat-ui/src/components/model-runner-modal/detector-runner.tsx b/cvat-ui/src/components/model-runner-modal/detector-runner.tsx index 9d733387c2a5..5c5e139b5601 100644 --- a/cvat-ui/src/components/model-runner-modal/detector-runner.tsx +++ b/cvat-ui/src/components/model-runner-modal/detector-runner.tsx @@ -179,13 +179,13 @@ function DetectorRunner(props: Props): JSX.Element { { isDetector && !!Object.keys(mapping).length && ( Object.keys(mapping).map((modelLabel: string) => ( - + {modelLabel} - + {mapping[modelLabel]} - + Date: Mon, 31 Aug 2020 11:09:46 +0300 Subject: [PATCH 23/26] Updated changelog, increased version --- CHANGELOG.md | 1 + cvat-ui/package-lock.json | 2 +- cvat-ui/package.json | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4d8600f506e..0f4efbf54c7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ability to configure email verification for new users () - Link to django admin page from UI () - Notification message when users use wrong browser () +- On the fly annotation using DL detectors () ### Changed - Shape coordinates are rounded to 2 digits in dumped annotations () diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index fbbd3cea5674..0553cd480105 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.8.4", + "version": "1.8.5", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index 1065c3f82efd..a9fea40c7e37 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.8.4", + "version": "1.8.5", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { From efe8e50180d623606e7202c4e1b2d66c69fc427d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 1 Sep 2020 11:53:43 +0300 Subject: [PATCH 24/26] Removed extra files --- .../saic-vul/fbrs/nuclio/function.yaml | 59 ------------ .../pytorch/saic-vul/fbrs/nuclio/main.py | 29 ------ .../saic-vul/fbrs/nuclio/model_handler.py | 89 ------------------- 3 files changed, 177 deletions(-) delete mode 100644 serverless/pytorch/saic-vul/fbrs/nuclio/function.yaml delete mode 100644 serverless/pytorch/saic-vul/fbrs/nuclio/main.py delete mode 100644 serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py diff --git a/serverless/pytorch/saic-vul/fbrs/nuclio/function.yaml b/serverless/pytorch/saic-vul/fbrs/nuclio/function.yaml deleted file mode 100644 index ba8156098d7a..000000000000 --- a/serverless/pytorch/saic-vul/fbrs/nuclio/function.yaml +++ /dev/null @@ -1,59 +0,0 @@ -metadata: - name: pth.saic-vul.fbrs - namespace: cvat - annotations: - name: FBRS - type: interactor - spec: - framework: pytorch - min_pos_points: 1 - -spec: - description: f-BRS Rethinking Backpropagating Refinement for Interactive Segmentation - runtime: "python:3.6" - handler: main:handler - eventTimeout: 30s - env: - - name: PYTHONPATH - value: /opt/nuclio/fbrs - - build: - image: cvat/pth.saic-vul.fbrs - baseImage: python:3.6.11 - - directives: - preCopy: - - kind: WORKDIR - value: /opt/nuclio - - kind: RUN - value: git clone https://github.com/saic-vul/fbrs_interactive_segmentation.git fbrs - - kind: WORKDIR - value: /opt/nuclio/fbrs - - kind: ENV - value: fileid=1Z9dQtpWVTobEdmUBntpUU0pJl-pEXUwR - - kind: ENV - value: filename=model.pth - - kind: RUN - value: curl -c ./cookie -s -L "https://drive.google.com/uc?export=download&id=${fileid}" - - kind: RUN - value: curl -Lb ./cookie "https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ./cookie`&id=${fileid}" -o ${filename} - - kind: RUN - value: apt update && apt install -y libgl1-mesa-glx - - kind: RUN - value: pip3 install -r requirements.txt - - kind: WORKDIR - value: /opt/nuclio - - triggers: - myHttpTrigger: - maxWorkers: 2 - kind: "http" - workerAvailabilityTimeoutMilliseconds: 10000 - attributes: - maxRequestBodySize: 33554432 # 32MB - - platform: - attributes: - restartPolicy: - name: always - maximumRetryCount: 3 diff --git a/serverless/pytorch/saic-vul/fbrs/nuclio/main.py b/serverless/pytorch/saic-vul/fbrs/nuclio/main.py deleted file mode 100644 index 1edacfa373db..000000000000 --- a/serverless/pytorch/saic-vul/fbrs/nuclio/main.py +++ /dev/null @@ -1,29 +0,0 @@ -import json -import base64 -from PIL import Image -import io -from model_handler import ModelHandler - -def init_context(context): - context.logger.info("Init context... 0%") - - model = ModelHandler() - setattr(context.user_data, 'model', model) - - context.logger.info("Init context...100%") - -def handler(context, event): - context.logger.info("call handler") - data = event.body - pos_points = data["points"] - neg_points = [] - threshold = data.get("threshold", 0.5) - buf = io.BytesIO(base64.b64decode(data["image"].encode('utf-8'))) - image = Image.open(buf) - - polygon = context.user_data.model.handle(image, pos_points, - neg_points, threshold) - return context.Response(body=json.dumps(polygon), - headers={}, - content_type='application/json', - status_code=200) diff --git a/serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py b/serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py deleted file mode 100644 index e4d86cdf4b58..000000000000 --- a/serverless/pytorch/saic-vul/fbrs/nuclio/model_handler.py +++ /dev/null @@ -1,89 +0,0 @@ -import torch -import numpy as np -from torchvision import transforms -import cv2 -import os - -from isegm.inference import clicker -from isegm.inference.predictors import get_predictor -from isegm.inference.utils import load_deeplab_is_model, load_hrnet_is_model -from isegm.inference.clicker import Clicker, Click - -def convert_mask_to_polygon(mask): - mask = np.array(mask, dtype=np.uint8) - cv2.normalize(mask, mask, 0, 255, cv2.NORM_MINMAX) - contours = None - if int(cv2.__version__.split('.')[0]) > 3: - contours = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[0] - else: - contours = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS)[1] - - contours = max(contours, key=lambda arr: arr.size) - if contours.shape.count(1): - contours = np.squeeze(contours) - if contours.size < 3 * 2: - raise Exception('Less then three point have been detected. Can not build a polygon.') - - polygon = [] - for point in contours: - polygon.append([int(point[0]), int(point[1])]) - - return polygon - -class ModelHandler: - def __init__(self): - torch.backends.cudnn.deterministic = True - base_dir = os.environ.get("MODEL_PATH", "/opt/nuclio/fbrs") - model_path = os.path.join(base_dir, "model.pth") - state_dict = torch.load(model_path, map_location='cpu') - - self.net = None - backbone = 'auto' - for k in state_dict.keys(): - if 'feature_extractor.stage2.0.branches' in k: - self.net = load_hrnet_is_model(state_dict, device, backbone, **kwargs) - break - - self.device = 'cuda' if torch.cuda.is_available() else 'cpu' - if self.net is None: - self.net = load_deeplab_is_model(state_dict, self.device, backbone) - self.net.to(self.device) - - def handle(self, image, pos_points, neg_points, threshold): - input_transform = transforms.Compose([ - transforms.ToTensor(), - transforms.Normalize([.485, .456, .406], [.229, .224, .225]) - ]) - - image_nd = input_transform(image).to(self.device) - result_mask = np.zeros(image_nd.shape[:2], dtype=np.uint16) - - clicker = Clicker() - for x, y in pos_points: - click = Click(is_positive=True, coords=(y, x)) - clicker.add_click(click) - - for x, y in neg_points: - click = Click(is_positive=False, coords=(y, x)) - clicker.add_click(click) - - predictor_params = { - 'brs_mode': 'f-BRS-B', - 'brs_opt_func_params': {'min_iou_diff': 0.001}, - 'lbfgs_params': {'maxfun': 20}, - 'predictor_params': {'max_size': 800, 'net_clicks_limit': 8}, - 'prob_thresh': threshold, - 'zoom_in_params': {'expansion_ratio': 1.4, 'skip_clicks': 1, 'target_size': 480}} - predictor = get_predictor(self.net, device=self.device, - **predictor_params) - predictor.set_input_image(image_nd) - - object_prob = predictor.get_prediction(clicker) - if self.device == 'cuda': - torch.cuda.empty_cache() - object_mask = object_prob > threshold - polygon = convert_mask_to_polygon(object_mask) - - return polygon - - From 3d6f13d5ea4ecf39beec907ad8541f1370f0a18d Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 1 Sep 2020 11:54:54 +0300 Subject: [PATCH 25/26] Removed extra code --- .../src/components/model-runner-modal/detector-runner.tsx | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cvat-ui/src/components/model-runner-modal/detector-runner.tsx b/cvat-ui/src/components/model-runner-modal/detector-runner.tsx index 5c5e139b5601..62f97025f226 100644 --- a/cvat-ui/src/components/model-runner-modal/detector-runner.tsx +++ b/cvat-ui/src/components/model-runner-modal/detector-runner.tsx @@ -203,14 +203,6 @@ function DetectorRunner(props: Props): JSX.Element { )} { isDetector && !!taskLabels.length && !!modelLabels.length && ( <> - {/* - - Model - - - Task - - */} {renderSelector( From d5e3da5e37047247679cf4205a7bff2f55514ebc Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 1 Sep 2020 16:24:56 +0300 Subject: [PATCH 26/26] Fixed changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd573360b81d..f0f2a7db7be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added password reset functionality () - Ability to work with data on the fly (https://github.com/opencv/cvat/pull/2007) - Annotation in process outline color wheel () +- On the fly annotation using DL detectors () ### Changed - UI models (like DEXTR) were redesigned to be more interactive () @@ -35,12 +36,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ability to configure email verification for new users () - Link to django admin page from UI () - Notification message when users use wrong browser () -- On the fly annotation using DL detectors () ### Changed - Shape coordinates are rounded to 2 digits in dumped annotations () - COCO format does not produce polygon points for bbox annotations () -- UI models (like DEXTR) were redesigned to be more interactive () ### Fixed - Issue loading openvino models for semi-automatic and automatic annotation ()