Quality control page (#8329)

changelog.d/20240826_093730_klakhov_support_quality_plugin.md

@@ -0,0 +1,14 @@
+### Changed
+
+- Moved quality control from `analytics` page to `quality control` page
+  (<https://github.com/cvat-ai/cvat/pull/8329>)
+
+### Removed
+
+- Quality report no longer available in CVAT community version
+  (<https://github.com/cvat-ai/cvat/pull/8329>)
+
+### Added
+
+- Quality management tab on `quality control` allows to enabling/disabling GT frames
+  (<https://github.com/cvat-ai/cvat/pull/8329>)

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "15.1.3",
+  "version": "15.2.0",
   "type": "module",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "src/api.ts",

cvat-core/src/annotations.ts

@@ -11,58 +11,80 @@ import AnnotationsHistory from './annotations-history';
 import { checkObjectType } from './common';
 import Project from './project';
 import { Task, Job } from './session';
-import { ScriptingError, ArgumentError } from './exceptions';
+import { ArgumentError } from './exceptions';
 import { getDeletedFrames } from './frames';
 import { JobType } from './enums';
 
-type WeakMapItem = { collection: AnnotationsCollection, saver: AnnotationsSaver, history: AnnotationsHistory };
-const jobCache = new WeakMap<Task | Job, WeakMapItem>();
-const taskCache = new WeakMap<Task | Job, WeakMapItem>();
+const jobCollectionCache = new WeakMap<Task | Job, { collection: AnnotationsCollection; saver: AnnotationsSaver; }>();
+const taskCollectionCache = new WeakMap<Task | Job, { collection: AnnotationsCollection; saver: AnnotationsSaver; }>();
 
-function getCache(sessionType): WeakMap<Task | Job, WeakMapItem> {
-    if (sessionType === 'task') {
-        return taskCache;
-    }
+// save history separately as not all history actions are related to annotations (e.g. delete, restore frame are not)
+const jobHistoryCache = new WeakMap<Task | Job, AnnotationsHistory>();
+const taskHistoryCache = new WeakMap<Task | Job, AnnotationsHistory>();
 
-    if (sessionType === 'job') {
-        return jobCache;
+function getCache(sessionType: 'task' | 'job'): {
+    collection: typeof jobCollectionCache;
+    history: typeof jobHistoryCache;
+} {
+    if (sessionType === 'task') {
+        return {
+            collection: taskCollectionCache,
+            history: taskHistoryCache,
+        };
     }
 
-    throw new ScriptingError(`Unknown session type was received ${sessionType}`);
+    return {
+        collection: jobCollectionCache,
+        history: jobHistoryCache,
+    };
 }
 
 class InstanceNotInitializedError extends Error {}
 
-function getSession(session): WeakMapItem {
+export function getCollection(session): AnnotationsCollection {
     const sessionType = session instanceof Task ? 'task' : 'job';
-    const cache = getCache(sessionType);
+    const { collection } = getCache(sessionType);
 
-    if (cache.has(session)) {
-        return cache.get(session);
+    if (collection.has(session)) {
+        return collection.get(session).collection;
     }
 
     throw new InstanceNotInitializedError(
         'Session has not been initialized yet. Call annotations.get() or annotations.clear({ reload: true }) before',
     );
 }
 
-export function getCollection(session): AnnotationsCollection {
-    return getSession(session).collection;
-}
-
 export function getSaver(session): AnnotationsSaver {
-    return getSession(session).saver;
+    const sessionType = session instanceof Task ? 'task' : 'job';
+    const { collection } = getCache(sessionType);
+
+    if (collection.has(session)) {
+        return collection.get(session).saver;
+    }
+
+    throw new InstanceNotInitializedError(
+        'Session has not been initialized yet. Call annotations.get() or annotations.clear({ reload: true }) before',
+    );
 }
 
 export function getHistory(session): AnnotationsHistory {
-    return getSession(session).history;
+    const sessionType = session instanceof Task ? 'task' : 'job';
+    const { history } = getCache(sessionType);
+
+    if (history.has(session)) {
+        return history.get(session);
+    }
+
+    const initiatedHistory = new AnnotationsHistory();
+    history.set(session, initiatedHistory);
+    return initiatedHistory;
 }
 
 async function getAnnotationsFromServer(session: Job | Task): Promise<void> {
     const sessionType = session instanceof Task ? 'task' : 'job';
     const cache = getCache(sessionType);
 
-    if (!cache.has(session)) {
+    if (!cache.collection.has(session)) {
         const serializedAnnotations = await serverProxy.annotations.getAnnotations(sessionType, session.id);
 
         // Get meta information about frames
@@ -74,7 +96,7 @@ async function getAnnotationsFromServer(session: Job | Task): Promise<void> {
         }
         frameMeta.deleted_frames = await getDeletedFrames(sessionType, session.id);
 
-        const history = new AnnotationsHistory();
+        const history = cache.history.has(session) ? cache.history.get(session) : new AnnotationsHistory();
         const collection = new AnnotationsCollection({
             labels: session.labels,
             history,
@@ -87,16 +109,21 @@ async function getAnnotationsFromServer(session: Job | Task): Promise<void> {
         // eslint-disable-next-line no-unsanitized/method
         collection.import(serializedAnnotations);
         const saver = new AnnotationsSaver(serializedAnnotations.version, collection, session);
-        cache.set(session, { collection, saver, history });
+        cache.collection.set(session, { collection, saver });
+        cache.history.set(session, history);
     }
 }
 
 export function clearCache(session): void {
     const sessionType = session instanceof Task ? 'task' : 'job';
     const cache = getCache(sessionType);
 
-    if (cache.has(session)) {
-        cache.delete(session);
+    if (cache.collection.has(session)) {
+        cache.collection.delete(session);
+    }
+
+    if (cache.history.has(session)) {
+        cache.history.delete(session);
     }
 }
 
@@ -125,7 +152,9 @@ export async function clearAnnotations(
         checkObjectType('reload', reload, 'boolean', null);
 
         if (reload) {
-            cache.delete(session);
+            cache.collection.delete(session);
+            // delete history as it may relate to objects from collection we deleted above
+            cache.history.delete(session);
             return getAnnotationsFromServer(session);
         }
     }

cvat-core/src/api-implementation.ts

@@ -37,7 +37,7 @@ import {
 import QualityReport from './quality-report';
 import QualityConflict, { ConflictSeverity } from './quality-conflict';
 import QualitySettings from './quality-settings';
-import { FramesMetaData } from './frames';
+import { getFramesMeta } from './frames';
 import AnalyticsReport from './analytics-report';
 import { listActions, registerAction, runActions } from './annotations-actions';
 import { convertDescriptions, getServerAPISchema } from './server-schema';
@@ -520,9 +520,8 @@ export default function implementAPI(cvat: CVATCore): CVATCore {
         const settings = await serverProxy.analytics.quality.settings.get(params);
         const schema = await getServerAPISchema();
         const descriptions = convertDescriptions(schema.components.schemas.QualitySettings.properties);
-        return new QualitySettings({
-            ...settings, descriptions,
-        });
+
+        return new QualitySettings({ ...settings, descriptions });
     });
     implementationMixin(cvat.analytics.performance.reports, async (filter: AnalyticsReportFilter) => {
         checkFilter(filter, {
@@ -557,12 +556,9 @@ export default function implementAPI(cvat: CVATCore): CVATCore {
         const params = fieldsToSnakeCase(body);
         await serverProxy.analytics.performance.calculate(params, onUpdate);
     });
-    implementationMixin(cvat.frames.getMeta, async (type, id) => {
-        const result = await serverProxy.frames.getMeta(type, id);
-        return new FramesMetaData({
-            ...result,
-            deleted_frames: Object.fromEntries(result.deleted_frames.map((_frame) => [_frame, true])),
-        });
+    implementationMixin(cvat.frames.getMeta, async (type: 'job' | 'task', id: number) => {
+        const result = await getFramesMeta(type, id);
+        return result;
     });
 
     return cvat;

cvat-core/src/api.ts

@@ -16,12 +16,16 @@ import Project from './project';
 import implementProject from './project-implementation';
 import { Attribute, Label } from './labels';
 import MLModel from './ml-model';
-import { FrameData } from './frames';
+import { FrameData, FramesMetaData } from './frames';
 import CloudStorage from './cloud-storage';
 import Organization from './organization';
 import Webhook from './webhook';
 import AnnotationGuide from './guide';
 import BaseSingleFrameAction from './annotations-actions';
+import QualityReport from './quality-report';
+import QualityConflict from './quality-conflict';
+import QualitySettings from './quality-settings';
+import AnalyticsReport from './analytics-report';
 import { Request } from './request';
 
 import * as enums from './enums';
@@ -416,6 +420,12 @@ function build(): CVATCore {
             Webhook,
             AnnotationGuide,
             BaseSingleFrameAction,
+            QualitySettings,
+            AnalyticsReport,
+            QualityConflict,
+            QualityReport,
+            Request,
+            FramesMetaData,
         },
         utils: {
             mask2Rle,

cvat-core/src/frames.ts

@@ -435,19 +435,27 @@ Object.defineProperty(FrameData.prototype.data, 'implementation', {
     writable: false,
 });
 
-async function getJobMeta(jobID: number): Promise<FramesMetaData> {
-    if (!frameMetaCache[jobID]) {
-        frameMetaCache[jobID] = serverProxy.frames.getMeta('job', jobID)
+export async function getFramesMeta(type: 'job' | 'task', id: number, forceReload = false): Promise<FramesMetaData> {
+    if (type === 'task') {
+        // we do not cache task meta currently. So, each new call will results to the server request
+        const result = await serverProxy.frames.getMeta('task', id);
+        return new FramesMetaData({
+            ...result,
+            deleted_frames: Object.fromEntries(result.deleted_frames.map((_frame) => [_frame, true])),
+        });
+    }
+    if (!(id in frameMetaCache) || forceReload) {
+        frameMetaCache[id] = serverProxy.frames.getMeta('job', id)
             .then((serverMeta) => new FramesMetaData({
                 ...serverMeta,
                 deleted_frames: Object.fromEntries(serverMeta.deleted_frames.map((_frame) => [_frame, true])),
             }))
             .catch((error) => {
-                delete frameMetaCache[jobID];
+                delete frameMetaCache[id];
                 throw error;
             });
     }
-    return frameMetaCache[jobID];
+    return frameMetaCache[id];
 }
 
 async function saveJobMeta(meta: FramesMetaData, jobID: number): Promise<FramesMetaData> {
@@ -588,7 +596,7 @@ export async function getFrame(
 ): Promise<FrameData> {
     if (!(jobID in frameDataCache)) {
         const blockType = chunkType === 'video' ? BlockType.MP4VIDEO : BlockType.ARCHIVE;
-        const meta = await getJobMeta(jobID);
+        const meta = await getFramesMeta('job', jobID);
 
         const mean = meta.frames.reduce((a, b) => a + b.width * b.height, 0) / meta.frames.length;
         const stdDev = Math.sqrt(
@@ -655,31 +663,32 @@ export async function getDeletedFrames(instanceType: 'job' | 'task', id): Promis
     throw new Exception(`getDeletedFrames is not implemented for ${instanceType}`);
 }
 
-export function deleteFrame(jobID: number, frame: number): void {
-    const { meta } = frameDataCache[jobID];
+export async function deleteFrame(jobID: number, frame: number): Promise<void> {
+    const meta = await frameMetaCache[jobID];
     meta.deletedFrames[frame] = true;
 }
 
-export function restoreFrame(jobID: number, frame: number): void {
-    const { meta } = frameDataCache[jobID];
+export async function restoreFrame(jobID: number, frame: number): Promise<void> {
+    const meta = await frameMetaCache[jobID];
     delete meta.deletedFrames[frame];
 }
 
-export async function patchMeta(jobID: number): Promise<void> {
-    const { meta } = frameDataCache[jobID];
+export async function patchMeta(jobID: number): Promise<FramesMetaData> {
+    const meta = await frameMetaCache[jobID];
     const updatedFields = meta.getUpdated();
 
     if (Object.keys(updatedFields).length) {
-        const newMeta = await saveJobMeta(meta, jobID);
-        frameDataCache[jobID].meta = newMeta;
+        frameMetaCache[jobID] = saveJobMeta(meta, jobID);
     }
+    const newMeta = await frameMetaCache[jobID];
+    return newMeta;
 }
 
 export async function findFrame(
     jobID: number, frameFrom: number, frameTo: number, filters: { offset?: number, notDeleted: boolean },
 ): Promise<number | null> {
     const offset = filters.offset || 1;
-    const meta = await getJobMeta(jobID);
+    const meta = await getFramesMeta('job', jobID);
 
     const sign = Math.sign(frameTo - frameFrom);
     const predicate = sign > 0 ? (frame) => frame <= frameTo : (frame) => frame >= frameTo;

cvat-core/src/index.ts

@@ -23,7 +23,7 @@ import ObjectState from './object-state';
 import MLModel from './ml-model';
 import Issue from './issue';
 import Comment from './comment';
-import { FrameData } from './frames';
+import { FrameData, FramesMetaData } from './frames';
 import CloudStorage from './cloud-storage';
 import Organization, { Invitation } from './organization';
 import Webhook from './webhook';
@@ -209,6 +209,12 @@ export default interface CVATCore {
         Webhook: typeof Webhook;
         AnnotationGuide: typeof AnnotationGuide;
         BaseSingleFrameAction: typeof BaseSingleFrameAction;
+        QualityReport: typeof QualityReport;
+        QualityConflict: typeof QualityConflict;
+        QualitySettings: typeof QualitySettings;
+        AnalyticsReport: typeof AnalyticsReport;
+        Request: typeof Request;
+        FramesMetaData: typeof FramesMetaData;
     };
     utils: {
         mask2Rle: typeof mask2Rle;