Significant memory optimization when working with masks

changelog.d/20231013_095118_boris_optimize_masks.md

@@ -0,0 +1,4 @@
+### Fixed <!-- pick one -->
+
+- Optimized huge memory consumption when working with masks in the interface
+  (<https://github.com/opencv/cvat/pull/6996>)

cvat-canvas/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-canvas",
-  "version": "2.17.6",
+  "version": "2.18.0",
   "description": "Part of Computer Vision Annotation Tool which presents its canvas library",
   "main": "src/canvas.ts",
   "scripts": {

cvat-canvas/src/typescript/canvasView.ts

@@ -1796,7 +1796,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
                 if (state.shapeType === 'mask') {
                     const { points } = state;
                     const [left, top, right, bottom] = points.slice(-4);
-                    const imageBitmap = expandChannels(255, 255, 255, points, 4);
+                    const imageBitmap = expandChannels(255, 255, 255, points);
                     imageDataToDataURL(imageBitmap, right - left + 1, bottom - top + 1,
                         (dataURL: string) => new Promise((resolve) => {
                             const img = document.createElement('img');
@@ -2893,7 +2893,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
         const colorization = this.getShapeColorization(state);
         const color = fabric.Color.fromHex(colorization.fill).getSource();
         const [left, top, right, bottom] = points.slice(-4);
-        const imageBitmap = expandChannels(color[0], color[1], color[2], points, 4);
+        const imageBitmap = expandChannels(color[0], color[1], color[2], points);
 
         const image = this.adoptedContent.image().attr({
             clientID: state.clientID,

cvat-canvas/src/typescript/groupHandler.ts

@@ -192,7 +192,7 @@ export class GroupHandlerImpl implements GroupHandler {
                 const { points } = objectState;
                 const colorRGB = [139, 0, 139];
                 const [left, top, right, bottom] = points.slice(-4);
-                const imageBitmap = expandChannels(colorRGB[0], colorRGB[1], colorRGB[2], points, 4);
+                const imageBitmap = expandChannels(colorRGB[0], colorRGB[1], colorRGB[2], points);
 
                 const bbox = shape.bbox();
                 const image = this.canvas.image().attr({

cvat-canvas/src/typescript/interactionHandler.ts

@@ -307,7 +307,7 @@ export class InteractionHandlerImpl implements InteractionHandler {
             this.selectize(true, this.drawnIntermediateShape, erroredShape);
         } else if (shapeType === 'mask') {
             const [left, top, right, bottom] = points.slice(-4);
-            const imageBitmap = expandChannels(255, 255, 255, points, 4);
+            const imageBitmap = expandChannels(255, 255, 255, points);
 
             const image = this.canvas.image().attr({
                 'color-rendering': 'optimizeQuality',

cvat-canvas/src/typescript/masksHandler.ts

@@ -10,7 +10,7 @@ import {
 import consts from './consts';
 import { DrawHandler } from './drawHandler';
 import {
-    PropType, computeWrappingBox, alphaChannelOnly, expandChannels, imageDataToDataURL,
+    PropType, computeWrappingBox, zipChannels, expandChannels, imageDataToDataURL,
 } from './shared';
 
 interface WrappingBBox {
@@ -348,12 +348,12 @@ export class MasksHandlerImpl implements MasksHandler {
                 const continueInserting = options.e.ctrlKey;
                 const wrappingBbox = this.getDrawnObjectsWrappingBox();
                 const imageData = this.imageDataFromCanvas(wrappingBbox);
-                const alpha = alphaChannelOnly(imageData);
-                alpha.push(wrappingBbox.left, wrappingBbox.top, wrappingBbox.right, wrappingBbox.bottom);
+                const rle = zipChannels(imageData);
+                rle.push(wrappingBbox.left, wrappingBbox.top, wrappingBbox.right, wrappingBbox.bottom);
 
                 this.onDrawDone({
                     shapeType: this.drawData.shapeType,
-                    points: alpha,
+                    points: rle,
                 }, Date.now() - this.startTimestamp, continueInserting, this.drawData);
 
                 if (!continueInserting) {
@@ -528,7 +528,7 @@ export class MasksHandlerImpl implements MasksHandler {
                 const { points } = drawData.initialState;
                 const color = fabric.Color.fromHex(this.getStateColor(drawData.initialState)).getSource();
                 const [left, top, right, bottom] = points.slice(-4);
-                const imageBitmap = expandChannels(color[0], color[1], color[2], points, 4);
+                const imageBitmap = expandChannels(color[0], color[1], color[2], points);
                 imageDataToDataURL(imageBitmap, right - left + 1, bottom - top + 1,
                     (dataURL: string) => new Promise((resolve) => {
                         fabric.Image.fromURL(dataURL, (image: fabric.Image) => {
@@ -547,28 +547,29 @@ export class MasksHandlerImpl implements MasksHandler {
                     }));
 
                 this.isInsertion = true;
-            } else if (!this.isDrawing) {
-                // initialize drawing pipeline if not started
-                this.isDrawing = true;
-                this.redraw = drawData.redraw || null;
+            } else {
+                this.updateBrushTools(drawData.brushTool);
+                if (!this.isDrawing) {
+                    // initialize drawing pipeline if not started
+                    this.isDrawing = true;
+                    this.redraw = drawData.redraw || null;
+                }
             }
 
             this.canvas.getElement().parentElement.style.display = 'block';
             this.startTimestamp = Date.now();
         }
 
-        this.updateBrushTools(drawData.brushTool);
-
         if (!drawData.enabled && this.isDrawing) {
             try {
                 if (this.drawnObjects.length) {
                     const wrappingBbox = this.getDrawnObjectsWrappingBox();
                     const imageData = this.imageDataFromCanvas(wrappingBbox);
-                    const alpha = alphaChannelOnly(imageData);
-                    alpha.push(wrappingBbox.left, wrappingBbox.top, wrappingBbox.right, wrappingBbox.bottom);
+                    const rle = zipChannels(imageData);
+                    rle.push(wrappingBbox.left, wrappingBbox.top, wrappingBbox.right, wrappingBbox.bottom);
                     this.onDrawDone({
                         shapeType: this.drawData.shapeType,
-                        points: alpha,
+                        points: rle,
                         ...(Number.isInteger(this.redraw) ? { clientID: this.redraw } : {}),
                     }, Date.now() - this.startTimestamp, drawData.continue, this.drawData);
                 }
@@ -600,7 +601,7 @@ export class MasksHandlerImpl implements MasksHandler {
                 const { points } = editData.state;
                 const color = fabric.Color.fromHex(this.getStateColor(editData.state)).getSource();
                 const [left, top, right, bottom] = points.slice(-4);
-                const imageBitmap = expandChannels(color[0], color[1], color[2], points, 4);
+                const imageBitmap = expandChannels(color[0], color[1], color[2], points);
                 imageDataToDataURL(imageBitmap, right - left + 1, bottom - top + 1,
                     (dataURL: string) => new Promise((resolve) => {
                         fabric.Image.fromURL(dataURL, (image: fabric.Image) => {
@@ -634,9 +635,9 @@ export class MasksHandlerImpl implements MasksHandler {
                 if (this.drawnObjects.length) {
                     const wrappingBbox = this.getDrawnObjectsWrappingBox();
                     const imageData = this.imageDataFromCanvas(wrappingBbox);
-                    const alpha = alphaChannelOnly(imageData);
-                    alpha.push(wrappingBbox.left, wrappingBbox.top, wrappingBbox.right, wrappingBbox.bottom);
-                    this.onEditDone(this.editData.state, alpha);
+                    const rle = zipChannels(imageData);
+                    rle.push(wrappingBbox.left, wrappingBbox.top, wrappingBbox.right, wrappingBbox.bottom);
+                    this.onEditDone(this.editData.state, rle);
                 }
             } finally {
                 this.releaseEdit();

cvat-canvas/src/typescript/shared.ts

@@ -383,24 +383,53 @@ export function imageDataToDataURL(
     }, 'image/png');
 }
 
-export function alphaChannelOnly(imageData: Uint8ClampedArray): number[] {
-    const alpha = new Array(imageData.length / 4);
+export function zipChannels(imageData: Uint8ClampedArray): number[] {
+    const rle = [];
+
+    let prev = 0;
+    let summ = 0;
     for (let i = 3; i < imageData.length; i += 4) {
-        alpha[Math.floor(i / 4)] = imageData[i] > 0 ? 1 : 0;
+        const alpha = imageData[i] > 0 ? 1 : 0;
+        if (prev !== alpha) {
+            rle.push(summ);
+            prev = alpha;
+            summ = 1;
+        } else {
+            summ++;
+        }
     }
-    return alpha;
+
+    rle.push(summ);
+    return rle;
 }
 
-export function expandChannels(r: number, g: number, b: number, alpha: number[], endOffset = 0): Uint8ClampedArray {
-    const imageBitmap = new Uint8ClampedArray((alpha.length - endOffset) * 4);
-    for (let i = 0; i < alpha.length - endOffset; i++) {
-        const val = alpha[i] ? 1 : 0;
-        imageBitmap[i * 4] = r;
-        imageBitmap[i * 4 + 1] = g;
-        imageBitmap[i * 4 + 2] = b;
-        imageBitmap[i * 4 + 3] = val * 255;
+export function expandChannels(r: number, g: number, b: number, encoded: number[]): Uint8ClampedArray {
+    function rle2Mask(rle: number[], width: number, height: number): Uint8ClampedArray {
+        const decoded = new Uint8ClampedArray(width * height * 4).fill(0);
+        const { length } = rle;
+        let decodedIdx = 0;
+        let value = 0;
+        let i = 0;
+
+        while (i < length - 4) {
+            let count = rle[i];
+            while (count > 0) {
+                decoded[decodedIdx + 0] = r;
+                decoded[decodedIdx + 1] = g;
+                decoded[decodedIdx + 2] = b;
+                decoded[decodedIdx + 3] = value * 255;
+                decodedIdx += 4;
+                count--;
+            }
+            i++;
+            value = Math.abs(value - 1);
+        }
+
+        return decoded;
     }
-    return imageBitmap;
+
+    const [left, top, right, bottom] = encoded.slice(-4);
+    return rle2Mask(encoded, right - left + 1, bottom - top + 1);
 }
 
 export type PropType<T, Prop extends keyof T> = T[Prop];

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "11.1.0",
+  "version": "12.0.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "src/api.ts",
   "scripts": {

cvat-core/src/annotations-collection.ts

@@ -13,7 +13,7 @@
 import { Label } from './labels';
 import { ArgumentError, ScriptingError } from './exceptions';
 import ObjectState from './object-state';
-import { mask2Rle, truncateMask } from './object-utils';
+import { cropMask } from './object-utils';
 import config from './config';
 import {
     HistoryActions, ShapeType, ObjectType, colors, Source,
@@ -778,7 +778,7 @@
             tags: [],
         };
 
         function convertAttributes(accumulator, attrID) {
             const specID = +attrID;
             const value = this.attributes[attrID];
 
@@ -844,11 +844,7 @@
                         occluded: state.occluded || false,
                         points: state.shapeType === 'mask' ? (() => {
                             const { width, height } = this.frameMeta[state.frame];
-                            const points = truncateMask(state.points, 0, width, height);
-                            const [left, top, right, bottom] = points.splice(-4);
-                            const rlePoints = mask2Rle(points);
-                            rlePoints.push(left, top, right, bottom);
-                            return rlePoints;
+                            return cropMask(state.points, width, height);
                         })() : state.points,
                         rotation: state.rotation || 0,
                         type: state.shapeType,

cvat-core/src/annotations-objects.ts

@@ -14,7 +14,7 @@
 import AnnotationHistory from './annotations-history';
 import {
     checkNumberOfPoints, attrsAsAnObject, checkShapeArea, mask2Rle, rle2Mask,
-    computeWrappingBox, findAngleDiff, rotatePoint, validateAttributeValue, truncateMask,
+    computeWrappingBox, findAngleDiff, rotatePoint, validateAttributeValue, cropMask,
 } from './object-utils';
 
 const defaultGroupColor = '#E0E0E0';
@@ -2093,7 +2093,7 @@
             return new ObjectState(this.get(frame));
         }
 
         const updateElements = (affectedElements, action, property: 'points' | 'occluded' | 'hidden' | 'lock') => {
             const undoSkeletonProperties = this.elements.map((element) => element[property]);
             const undoSource = this.source;
             const redoSource = this.readOnlyFields.includes('source') ? this.source : computeNewSource(this.source);
@@ -2201,8 +2201,7 @@
         Annotation.prototype.validateStateBeforeSave.call(this, data, updated);
         if (updated.points) {
             const { width, height } = this.frameMeta[frame];
-            const fittedPoints = truncateMask(data.points, 0, width, height);
-            return fittedPoints;
+            return cropMask(data.points, width, height);
         }
 
         return [];
@@ -2264,7 +2263,7 @@
         const undoSource = this.source;
 
         const [redoLeft, redoTop, redoRight, redoBottom] = maskPoints.splice(-4);
-        const points = mask2Rle(maskPoints);
+        const points = maskPoints;
 
         const redoPoints = points;
         const redoSource = computeNewSource(this.source);
@@ -2301,16 +2300,26 @@
         }
     }
 
-    static distance(points: number[], x: number, y: number): null | number {
-        const [left, top, right, bottom] = points.slice(-4);
+    static distance(rle: number[], x: number, y: number): null | number {
+        const [left, top, right, bottom] = rle.slice(-4);
         const [width, height] = [right - left + 1, bottom - top + 1];
         const [translatedX, translatedY] = [x - left, y - top];
         if (translatedX < 0 || translatedX >= width || translatedY < 0 || translatedY >= height) {
             return null;
         }
+
         const offset = Math.floor(translatedY) * width + Math.floor(translatedX);
+        let sum = 0;
+        let value = 0;
+
+        for (const count of rle) {
+            sum += count;
+            if (sum > offset) {
+                return value || null;
+            }
+            value = Math.abs(value - 1);
+        }
 
-        if (points[offset]) return 1;
         return null;
     }
 }
@@ -2324,7 +2333,7 @@
 
 MaskShape.prototype.get = function (frame) {
     const result = Shape.prototype.get.call(this, frame);
-    result.points = rle2Mask(this.points, this.right - this.left + 1, this.bottom - this.top + 1);
+    result.points = this.points.slice(0);
     result.points.push(this.left, this.top, this.right, this.bottom);
     return result;
 };
@@ -2486,7 +2495,7 @@
             return updatedMatching;
         }
 
         function reduceInterpolation(interpolatedPoints, matching, leftPoints, rightPoints) {
             function averagePoint(points: Point2D[]): Point2D {
                 let sumX = 0;
                 let sumY = 0;
@@ -2505,7 +2514,7 @@
                 return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);
             }
 
             function minimizeSegment(baseLength: number, N: number, startInterpolated, stopInterpolated) {
                 const threshold = baseLength / (2 * N);
                 const minimized = [interpolatedPoints[startInterpolated]];
                 let latestPushed = startInterpolated;
@@ -2542,7 +2551,7 @@
                 interpolatedIndexes[i] = matching[i].map(() => accumulated++);
             }
 
             function leftSegment(start, stop) {
                 const startInterpolated = interpolatedIndexes[start][0];
                 const stopInterpolated = interpolatedIndexes[stop][0];
 
@@ -2557,7 +2566,7 @@
                 reduced.push(...minimizeSegment(baseLength, N, startInterpolated, stopInterpolated));
             }
 
             function rightSegment(leftPoint) {
                 const start = matching[leftPoint][0];
                 const [stop] = matching[leftPoint].slice(-1);
                 const startInterpolated = interpolatedIndexes[leftPoint][0];

cvat-core/src/api.ts

@@ -28,6 +28,7 @@ import {
     Exception, ArgumentError, DataError, ScriptingError, ServerError,
 } from './exceptions';
 
+import { mask2Rle, rle2Mask } from './object-utils';
 import User from './user';
 import pjson from '../package.json';
 import config from './config';
@@ -320,6 +321,10 @@ function build() {
             Webhook,
             AnnotationGuide,
         },
+        utils: {
+            mask2Rle,
+            rle2Mask,
+        },
     };
 
     cvat.server = Object.freeze(cvat.server);
@@ -340,8 +345,8 @@ function build() {
     cvat.organizations = Object.freeze(cvat.organizations);
     cvat.webhooks = Object.freeze(cvat.webhooks);
     cvat.analytics = Object.freeze(cvat.analytics);
-    cvat.storage = Object.freeze(cvat.storage);
     cvat.classes = Object.freeze(cvat.classes);
+    cvat.utils = Object.freeze(cvat.utils);
 
     const implemented = Object.freeze(implementAPI(cvat));
     return implemented;

cvat-core/src/exceptions.ts

@@ -25,11 +25,6 @@ export class Exception extends Error {
         const line = info.lineNumber;
         const column = info.columnNumber;
 
-        // TODO: NOT IMPLEMENTED?
-        // const {
-        //     jobID, taskID, clientID, projID,
-        // } = config;
-
         Object.defineProperties(
             this,
             Object.freeze({
@@ -63,18 +58,6 @@ export class Exception extends Error {
                      */
                     get: () => time,
                 },
-                // jobID: {
-                //     get: () => jobID,
-                // },
-                // taskID: {
-                //     get: () => taskID,
-                // },
-                // projID: {
-                //     get: () => projID,
-                // },
-                // clientID: {
-                //     get: () => clientID,
-                // },
                 filename: {
                     /**
                      * @name filename