Intelligent scissors with OpenCV javascript (#2689)

* Some UI implementations * Added opencv wrapper * Updated Opencv wrapper * Moved initialization stub * Added threshold * Setup interaction with canvas * Fixed couple of issues * Added threshold, changing size via ctrl * tmp * Aborted host change * Fixed threshold * Aborted host * Some fixes * Using ready label selector * Raw implementation * Added additional arguments * Fixed some minor issues * Removed unused file * Fixed tool reset * Added short instructions to update opencv.js * Fixed corner case * Added error handler, opencv version, updated cvat_proxy & mod_wsgi * OpenCV returned back * Using dinamic function instead of script * Updated changelog & version
cvat-ai · Feb 3, 2021 · e0fc323 · e0fc323
1 parent 3d4fad4
commit e0fc323
Show file tree

Hide file tree

Showing 35 changed files with 1,073 additions and 87 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - CVAT-3D: support lidar data on the server side (<https://github.com/openvinotoolkit/cvat/pull/2534>)
+- Intelligent scissors with OpenCV javascript (<https://github.com/openvinotoolkit/cvat/pull/2689>)
 
 ### Changed
 

diff --git a/cvat-canvas/package-lock.json b/cvat-canvas/package-lock.json
diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json
@@ -1,6 +1,6 @@
 {
  "name": "cvat-canvas",
- "version": "2.2.2",
+ "version": "2.3.0",
  "description": "Part of Computer Vision Annotation Tool which presents its canvas library",
  "main": "src/canvas.ts",
  "scripts": {

diff --git a/cvat-canvas/src/scss/canvas.scss b/cvat-canvas/src/scss/canvas.scss
@@ -51,6 +51,10 @@ polyline.cvat_shape_drawing_opacity {
  stroke: red;
 }
 
+.cvat_canvas_threshold {
+ stroke: red;
+}
+
 .cvat_canvas_shape_grouping {
  @extend .cvat_shape_action_dasharray;
  @extend .cvat_shape_action_opacity;

diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2019-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -76,6 +76,10 @@ export interface InteractionData {
  crosshair?: boolean;
  minPosVertices?: number;
  minNegVertices?: number;
+ enableNegVertices?: boolean;
+ enableThreshold?: boolean;
+ enableSliding?: boolean;
+ allowRemoveOnlyLast?: boolean;
 }
 
 export interface InteractionResult {

diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts
@@ -165,6 +165,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
  shapes: InteractionResult[] | null,
  shapesUpdated: boolean = true,
  isDone: boolean = false,
+ threshold: number | null = null,
  ): void {
  const { zLayer } = this.controller;
  if (Array.isArray(shapes)) {
@@ -176,6 +177,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
  isDone,
  shapes,
  zOrder: zLayer || 0,
+ threshold,
  },
  });
 
@@ -1050,6 +1052,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
  });
 
  this.content.addEventListener('wheel', (event): void => {
+ if (event.ctrlKey) return;
  const { offset } = this.controller.geometry;
  const point = translateToSVG(this.content, [event.clientX, event.clientY]);
  self.controller.zoom(point[0] - offset, point[1] - offset, event.deltaY > 0 ? -1 : 1);

diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts
@@ -1,4 +1,4 @@
-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -24,6 +24,8 @@ export class InteractionHandlerImpl implements InteractionHandler {
  private interactionShapes: SVG.Shape[];
  private currentInteractionShape: SVG.Shape | null;
  private crosshair: Crosshair;
+ private threshold: SVG.Rect | null;
+ private thresholdRectSize: number;
 
  private prepareResult(): InteractionResult[] {
  return this.interactionShapes.map(
@@ -63,12 +65,21 @@ export class InteractionHandlerImpl implements InteractionHandler {
  return enabled && !ctrlKey && !!interactionShapes.length;
  }
 
- const minimumVerticesAchieved =
-  (typeof minPosVertices === 'undefined' || minPosVertices <= positiveShapes.length) &&
-  (typeof minNegVertices === 'undefined' || minPosVertices <= negativeShapes.length);
+ const minPosVerticesAchieved = typeof minPosVertices === 'undefined' || minPosVertices <= positiveShapes.length;
+ const minNegVerticesAchieved = typeof minNegVertices === 'undefined' || minPosVertices <= negativeShapes.length;
+ const minimumVerticesAchieved = minPosVerticesAchieved && minNegVerticesAchieved;
  return enabled && !ctrlKey && minimumVerticesAchieved && shapesWereUpdated;
  }
 
+ private addThreshold(): void {
+ const { x, y } = this.cursorPosition;
+ this.threshold = this.canvas
+ .rect(this.thresholdRectSize, this.thresholdRectSize)
+ .fill('none')
+ .addClass('cvat_canvas_threshold');
+ this.threshold.center(x, y);
+ }
+
  private addCrosshair(): void {
  const { x, y } = this.cursorPosition;
  this.crosshair.show(this.canvas, x, y, this.geometry.scale);
@@ -80,9 +91,12 @@ export class InteractionHandlerImpl implements InteractionHandler {
 
  private interactPoints(): void {
  const eventListener = (e: MouseEvent): void => {
- if ((e.button === 0 || e.button === 2) && !e.altKey) {
+ if ((e.button === 0 || (e.button === 2 && this.interactionData.enableNegVertices)) && !e.altKey) {
  e.preventDefault();
  const [cx, cy] = translateToSVG((this.canvas.node as any) as SVGSVGElement, [e.clientX, e.clientY]);
+ if (!this.isWithingFrame(cx, cy)) return;
+ if (!this.isWithinThreshold(cx, cy)) return;
+
  this.currentInteractionShape = this.canvas
  .circle((consts.BASE_POINT_SIZE * 2) / this.geometry.scale)
  .center(cx, cy)
@@ -101,6 +115,12 @@ export class InteractionHandlerImpl implements InteractionHandler {
 
  const self = this.currentInteractionShape;
  self.on('mouseenter', (): void => {
+ if (this.interactionData.allowRemoveOnlyLast) {
+ if (this.interactionShapes.indexOf(self) !== this.interactionShapes.length - 1) {
+ return;
+ }
+ }
+
  self.attr({
  'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / this.geometry.scale,
  });
@@ -166,6 +186,10 @@ export class InteractionHandlerImpl implements InteractionHandler {
  if (this.interactionData.crosshair) {
  this.addCrosshair();
  }
+
+ if (this.interactionData.enableThreshold) {
+ this.addThreshold();
+ }
  }
 
  private startInteraction(): void {
@@ -183,6 +207,11 @@ export class InteractionHandlerImpl implements InteractionHandler {
  this.removeCrosshair();
  }
 
+ if (this.threshold) {
+ this.threshold.remove();
+ this.threshold = null;
+ }
+
  this.canvas.off('mousedown.interaction');
  this.interactionShapes.forEach((shape: SVG.Shape): SVG.Shape => shape.remove());
  this.interactionShapes = [];
@@ -192,14 +221,39 @@ export class InteractionHandlerImpl implements InteractionHandler {
  }
  }
 
+ private isWithinThreshold(x: number, y: number): boolean {
+ const [prev] = this.interactionShapes.slice(-1);
+ if (!this.interactionData.enableThreshold || !prev) {
+ return true;
+ }
+
+ const [prevCx, prevCy] = [(prev as SVG.Circle).cx(), (prev as SVG.Circle).cy()];
+ const xDiff = Math.abs(prevCx - x);
+ const yDiff = Math.abs(prevCy - y);
+
+ return xDiff < this.thresholdRectSize / 2 && yDiff < this.thresholdRectSize / 2;
+ }
+
+ private isWithingFrame(x: number, y: number): boolean {
+ const { offset, image } = this.geometry;
+ const { width, height } = image;
+ const [imageX, imageY] = [Math.round(x - offset), Math.round(y - offset)];
+ return imageX >= 0 && imageX < width && imageY >= 0 && imageY < height;
+ }
+
  public constructor(
- onInteraction: (shapes: InteractionResult[] | null, shapesUpdated?: boolean, isDone?: boolean) => void,
+ onInteraction: (
+ shapes: InteractionResult[] | null,
+ shapesUpdated?: boolean,
+ isDone?: boolean,
+ threshold?: number,
+ ) => void,
  canvas: SVG.Container,
  geometry: Geometry,
  ) {
  this.onInteraction = (shapes: InteractionResult[] | null, shapesUpdated?: boolean, isDone?: boolean): void => {
  this.shapesWereUpdated = false;
- onInteraction(shapes, shapesUpdated, isDone);
+ onInteraction(shapes, shapesUpdated, isDone, this.threshold ? this.thresholdRectSize / 2 : null);
  };
  this.canvas = canvas;
  this.geometry = geometry;
@@ -208,6 +262,8 @@ export class InteractionHandlerImpl implements InteractionHandler {
  this.interactionData = { enabled: false };
  this.currentInteractionShape = null;
  this.crosshair = new Crosshair();
+ this.threshold = null;
+ this.thresholdRectSize = 300;
  this.cursorPosition = {
  x: 0,
  y: 0,
@@ -219,6 +275,43 @@ export class InteractionHandlerImpl implements InteractionHandler {
  if (this.crosshair) {
  this.crosshair.move(x, y);
  }
+ if (this.threshold) {
+ this.threshold.center(x, y);
+ }
+
+ if (this.interactionData.enableSliding && this.interactionShapes.length) {
+ if (this.isWithingFrame(x, y)) {
+ if (this.interactionData.enableThreshold && !this.isWithinThreshold(x, y)) return;
+ this.onInteraction(
+ [
+ ...this.prepareResult(),
+ {
+ points: [x - this.geometry.offset, y - this.geometry.offset],
+ shapeType: 'points',
+ button: 0,
+ },
+ ],
+ true,
+ false,
+ );
+ }
+ }
+ });
+
+ this.canvas.on('wheel.interaction', (e: WheelEvent): void => {
+ if (e.ctrlKey) {
+ if (this.threshold) {
+ const { x, y } = this.cursorPosition;
+ e.preventDefault();
+ if (e.deltaY > 0) {
+ this.thresholdRectSize *= 6 / 5;
+ } else {
+ this.thresholdRectSize *= 5 / 6;
+ }
+ this.threshold.size(this.thresholdRectSize, this.thresholdRectSize);
+ this.threshold.center(x, y);
+ }
+ }
  });
 
  document.body.addEventListener('keyup', (e: KeyboardEvent): void => {

diff --git a/cvat-core/src/ml-model.js b/cvat-core/src/ml-model.js
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2019-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -17,6 +17,7 @@ class MLModel {
  this._params = {
  canvas: {
  minPosVertices: data.min_pos_points,
+ enableNegVertices: true,
  },
  };
  }

diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json
diff --git a/cvat-ui/package.json b/cvat-ui/package.json
@@ -1,6 +1,6 @@
 {
  "name": "cvat-ui",
- "version": "1.13.8",
+ "version": "1.14.0",
  "description": "CVAT single-page application",
  "main": "src/index.tsx",
  "scripts": {

diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts
@@ -18,6 +18,7 @@ import {
  ContextMenuType,
  Workspace,
  Model,
+ OpenCVTool,
 } from 'reducers/interfaces';
 
 import getCore from 'cvat-core-wrapper';
@@ -1354,7 +1355,10 @@ export function pasteShapeAsync(): ThunkAction {
  };
 }
 
-export function interactWithCanvas(activeInteractor: Model, activeLabelID: number): AnyAction {
+export function interactWithCanvas(
+ activeInteractor: Model | OpenCVTool,
+ activeLabelID: number,
+): AnyAction {
  return {
  type: AnnotationActionTypes.INTERACT_WITH_CANVAS,
  payload: {

diff --git a/cvat-ui/src/assets/opencv.svg b/cvat-ui/src/assets/opencv.svg
diff --git a/...src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx b/...src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx
@@ -15,6 +15,7 @@ import MoveControl from './move-control';
 import FitControl from './fit-control';
 import ResizeControl from './resize-control';
 import ToolsControl from './tools-control';
+import OpenCVControl from './opencv-control';
 import DrawRectangleControl from './draw-rectangle-control';
 import DrawPolygonControl from './draw-polygon-control';
 import DrawPolylineControl from './draw-polyline-control';
@@ -90,6 +91,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
  ActiveControl.DRAW_RECTANGLE,
  ActiveControl.DRAW_CUBOID,
  ActiveControl.AI_TOOLS,
+ ActiveControl.OPENCV_TOOLS,
  ].includes(activeControl);
 
  if (!drawing) {
@@ -103,7 +105,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
  repeatDrawShape();
  }
  } else {
- if (activeControl === ActiveControl.AI_TOOLS) {
+ if ([ActiveControl.AI_TOOLS, ActiveControl.OPENCV_TOOLS].includes(activeControl)) {
  // separated API method
  canvasInstance.interact({ enabled: false });
  return;
@@ -187,6 +189,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
 
  <hr />
  <ToolsControl />
+ <OpenCVControl />
  <DrawRectangleControl
  canvasInstance={canvasInstance}
  isDrawing={activeControl === ActiveControl.DRAW_RECTANGLE}