Support of context images for 2D tasks

CHANGELOG.md

@@ -9,11 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
--
+- Support of context images for 2D image tasks (<https://github.com/openvinotoolkit/cvat/pull/3122>)
 
 ### Changed
 
--
+- Updated manifest format, added meta with related images (<https://github.com/openvinotoolkit/cvat/pull/3122>)
 
 ### Deprecated
 
@@ -111,7 +111,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed filters select overflow (<https://github.com/openvinotoolkit/cvat/pull/2614>)
 - Fixed tasks in project auto annotation (<https://github.com/openvinotoolkit/cvat/pull/2725>)
 - Cuboids are missed in annotations statistics (<https://github.com/openvinotoolkit/cvat/pull/2704>)
-- The list of files attached to the task is not displayed (<https://github.com/openvinotoolkit/cvat/pul
+- The list of files attached to the task is not displayed (<https://github.com/openvinotoolkit/cvat/pull/2706>)
 - A couple of css-related issues (top bar disappear, wrong arrow position on collapse elements) (<https://github.com/openvinotoolkit/cvat/pull/2736>)
 - Issue with point region doesn't work in Firefox (<https://github.com/openvinotoolkit/cvat/pull/2727>)
 - Fixed cuboid perspective change (<https://github.com/openvinotoolkit/cvat/pull/2733>)

cvat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-core",
-  "version": "3.12.1",
+  "version": "3.13.0",
   "description": "Part of Computer Vision Tool which presents an interface for client-side integration",
   "main": "babel.config.js",
   "scripts": {

cvat-core/src/frames.js

@@ -19,7 +19,15 @@
      */
     class FrameData {
         constructor({
-            width, height, name, taskID, frameNumber, startFrame, stopFrame, decodeForward,
+            width,
+            height,
+            name,
+            taskID,
+            frameNumber,
+            startFrame,
+            stopFrame,
+            decodeForward,
+            has_related_context: hasRelatedContext,
         }) {
             Object.defineProperties(
                 this,
@@ -72,6 +80,18 @@
                         value: frameNumber,
                         writable: false,
                     },
+                    /**
+                     * True if some context images are associated with this frame
+                     * @name hasRelatedContext
+                     * @type {boolean}
+                     * @memberof module:API.cvat.classes.FrameData
+                     * @readonly
+                     * @instance
+                     */
+                    hasRelatedContext: {
+                        value: hasRelatedContext,
+                        writable: false,
+                    },
                     startFrame: {
                         value: startFrame,
                         writable: false,

cvat-core/src/server-proxy.js

@@ -756,11 +756,7 @@
                         },
                     );
                 } catch (errorData) {
-                    const code = errorData.response ? errorData.response.status : errorData.code;
-                    throw new ServerError(
-                        `Could not get Image Context of the frame for the task ${tid} from the server`,
-                        code,
-                    );
+                    throw generateError(errorData);
                 }
 
                 return response.data;

cvat-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.19.1",
+  "version": "1.20.0",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

cvat-ui/src/actions/annotation-actions.ts

@@ -196,6 +196,8 @@ export enum AnnotationActionTypes {
     GET_PREDICTIONS_SUCCESS = 'GET_PREDICTIONS_SUCCESS',
     HIDE_SHOW_CONTEXT_IMAGE = 'HIDE_SHOW_CONTEXT_IMAGE',
     GET_CONTEXT_IMAGE = 'GET_CONTEXT_IMAGE',
+    GET_CONTEXT_IMAGE_SUCCESS = 'GET_CONTEXT_IMAGE_SUCCESS',
+    GET_CONTEXT_IMAGE_FAILED = 'GET_CONTEXT_IMAGE_FAILED',
 }
 
 export function saveLogsAsync(): ThunkAction {
@@ -715,6 +717,7 @@ export function changeFrameAsync(toFrame: number, fillBuffer?: boolean, frameSte
                         number: state.annotation.player.frame.number,
                         data: state.annotation.player.frame.data,
                         filename: state.annotation.player.frame.filename,
+                        hasRelatedContext: state.annotation.player.frame.hasRelatedContext,
                         delay: state.annotation.player.frame.delay,
                         changeTime: state.annotation.player.frame.changeTime,
                         states: state.annotation.annotations.states,
@@ -766,6 +769,7 @@ export function changeFrameAsync(toFrame: number, fillBuffer?: boolean, frameSte
                     number: toFrame,
                     data,
                     filename: data.filename,
+                    hasRelatedContext: data.hasRelatedContext,
                     states,
                     minZ,
                     maxZ,
@@ -1031,6 +1035,7 @@ export function getJobAsync(tid: number, jid: number, initialFrame: number, init
                     states,
                     frameNumber,
                     frameFilename: frameData.filename,
+                    frameHasRelatedContext: frameData.hasRelatedContext,
                     frameData,
                     colors,
                     filters,
@@ -1636,31 +1641,23 @@ export function getContextImage(): ThunkAction {
     return async (dispatch: ActionCreator<Dispatch>): Promise<void> => {
         const state: CombinedState = getStore().getState();
         const { instance: job } = state.annotation.job;
-        const { frame, contextImage } = state.annotation.player;
+        const { number: frameNumber } = state.annotation.player.frame;
 
         try {
-            const context = await job.frames.contextImage(job.task.id, frame.number);
-            const loaded = true;
-            const contextImageHide = contextImage.hidden;
             dispatch({
                 type: AnnotationActionTypes.GET_CONTEXT_IMAGE,
-                payload: {
-                    context,
-                    loaded,
-                    contextImageHide,
-                },
+                payload: {},
+            });
+
+            const contextImageData = await job.frames.contextImage(job.task.id, frameNumber);
+            dispatch({
+                type: AnnotationActionTypes.GET_CONTEXT_IMAGE_SUCCESS,
+                payload: { contextImageData },
             });
         } catch (error) {
-            const context = '';
-            const loaded = true;
-            const contextImageHide = contextImage.hidden;
             dispatch({
-                type: AnnotationActionTypes.GET_CONTEXT_IMAGE,
-                payload: {
-                    context,
-                    loaded,
-                    contextImageHide,
-                },
+                type: AnnotationActionTypes.GET_CONTEXT_IMAGE_FAILED,
+                payload: { error },
             });
         }
     };

cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx

@@ -18,6 +18,7 @@ import getCore from 'cvat-core-wrapper';
 import consts from 'consts';
 import CVATTooltip from 'components/common/cvat-tooltip';
 import ImageSetupsContent from './image-setups-content';
+import ContextImage from '../standard-workspace/context-image/context-image';
 
 const cvat = getCore();
 
@@ -773,12 +774,12 @@ export default class CanvasWrapperComponent extends React.PureComponent<Props> {
             maxZLayer,
             curZLayer,
             minZLayer,
-            onSwitchZLayer,
-            onAddZLayer,
             keyMap,
             switchableAutomaticBordering,
             automaticBordering,
             onSwitchAutomaticBordering,
+            onSwitchZLayer,
+            onAddZLayer,
         } = this.props;
 
         const preventDefault = (event: KeyboardEvent | undefined): void => {
@@ -817,6 +818,8 @@ export default class CanvasWrapperComponent extends React.PureComponent<Props> {
                     }}
                 />
 
+                <ContextImage />
+
                 <Dropdown trigger='click' placement='topCenter' overlay={<ImageSetupsContent />}>
                     <UpOutlined className='cvat-canvas-image-setups-trigger' />
                 </Dropdown>

cvat-ui/src/components/annotation-page/canvas/canvas-wrapper3D.tsx

@@ -14,20 +14,16 @@ import { Workspace } from 'reducers/interfaces';
 import {
     CAMERA_ACTION, Canvas3d, MouseInteraction, ViewType,
 } from 'cvat-canvas3d-wrapper';
-import ContextImage from '../standard3D-workspace/context-image/context-image';
-import CVATTooltip from '../../common/cvat-tooltip';
+import ContextImage from 'components/annotation-page/standard-workspace/context-image/context-image';
+import CVATTooltip from 'components/common/cvat-tooltip';
 
 interface Props {
     canvasInstance: Canvas3d;
     jobInstance: any;
     frameData: any;
     curZLayer: number;
-    contextImageHide: boolean;
-    loaded: boolean;
-    data: string;
     annotations: any[];
     onSetupCanvas: () => void;
-    getContextImage(): void;
     onResetCanvas(): void;
     workspace: Workspace;
     animateID: any;
@@ -119,9 +115,7 @@ const CanvasWrapperComponent = (props: Props): ReactElement => {
     const sideView = useRef<HTMLDivElement | null>(null);
     const frontView = useRef<HTMLDivElement | null>(null);
 
-    const {
-        frameData, contextImageHide, getContextImage, loaded, data, annotations, curZLayer,
-    } = props;
+    const { frameData, annotations, curZLayer } = props;
 
     const onCanvasSetup = (): void => {
         const { onSetupCanvas } = props;
@@ -345,13 +339,7 @@ const CanvasWrapperComponent = (props: Props): ReactElement => {
 
     return (
         <Layout.Content className='cvat-canvas3d-fullsize' id='canvas3d-container'>
-            <ContextImage
-                frame={frameData}
-                contextImageHide={contextImageHide}
-                getContextImage={getContextImage}
-                loaded={loaded}
-                data={data}
-            />
+            <ContextImage />
             <ResizableBox
                 className='cvat-resizable'
                 width={Infinity}

cvat-ui/src/components/annotation-page/standard-workspace/context-image/context-image.tsx

@@ -0,0 +1,74 @@
+// Copyright (C) 2021 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+
+import React, { useEffect, useState } from 'react';
+import { notification } from 'antd';
+import { useDispatch, useSelector } from 'react-redux';
+import { QuestionCircleOutlined, ShrinkOutlined } from '@ant-design/icons';
+import Spin from 'antd/lib/spin';
+import Image from 'antd/lib/image';
+
+import { CombinedState } from 'reducers/interfaces';
+import { hideShowContextImage, getContextImage } from 'actions/annotation-actions';
+import CVATTooltip from 'components/common/cvat-tooltip';
+
+export default function ContextImage(): JSX.Element | null {
+    const dispatch = useDispatch();
+    const { number: frame, hasRelatedContext } = useSelector((state: CombinedState) => state.annotation.player.frame);
+    const { data: contextImageData, hidden: contextImageHidden, fetching: contextImageFetching } = useSelector(
+        (state: CombinedState) => state.annotation.player.contextImage,
+    );
+    const [requested, setRequested] = useState(false);
+
+    useEffect(() => {
+        if (requested) {
+            setRequested(false);
+        }
+    }, [frame]);
+
+    useEffect(() => {
+        if (hasRelatedContext && !contextImageHidden && !requested) {
+            dispatch(getContextImage());
+            setRequested(true);
+        }
+    }, [contextImageHidden, requested, hasRelatedContext]);
+
+    if (!hasRelatedContext) {
+        return null;
+    }
+
+    return (
+        <div className='cvat-context-image-wrapper' {...(contextImageHidden ? { style: { width: '32px' } } : {})}>
+            <div className='cvat-context-image-wrapper-header' />
+            {contextImageFetching ? <Spin size='small' /> : null}
+            {contextImageHidden ? (
+                <CVATTooltip title='A context image is available'>
+                    <QuestionCircleOutlined
+                        className='cvat-context-image-switcher'
+                        onClick={() => dispatch(hideShowContextImage(false))}
+                    />
+                </CVATTooltip>
+            ) : (
+                <>
+                    <ShrinkOutlined
+                        className='cvat-context-image-switcher'
+                        onClick={() => dispatch(hideShowContextImage(true))}
+                    />
+                    <Image
+                        {...(contextImageData ? { src: contextImageData } : {})}
+                        onError={() => {
+                            notification.error({
+                                message: 'Could not display context image',
+                                description: `Source is  ${
+                                    contextImageData === null ? 'empty' : contextImageData.slice(0, 100)
+                                }`,
+                            });
+                        }}
+                        className='cvat-context-image'
+                    />
+                </>
+            )}
+        </div>
+    );
+}

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/control-visibility-observer.tsx

@@ -41,7 +41,7 @@ export function ExtraControlsControl(): JSX.Element {
         >
             <SmallDashOutlined
                 style={{ visibility: hasChildren ? 'visible' : 'hidden' }}
-                className='cvat-extra-controls-control'
+                className='cvat-extra-controls-control cvat-antd-icon-control'
             />
         </Popover>
     );