CVAT-3D: support lidar data on the server side (#2534)

* CVAT-3D Updated the Mime Types with Bin Support, added dependency of open3D * CVAT-3D Added additional column as Dimension for engine_task table and created a relatedfiles table for PCD to Image mapping. * Added Support for 3D file Upload in BIN and PCD. * Added Dimension attribute defaulting to 2D for importer and exporter. * Added props passing for dimension attribute, filtering of import, Migration Scripts and Dimension attribute for MpegChunk Writers Co-authored-by: cdp <cdp123>
cvat-ai · Jan 8, 2021 · 069fadc · 069fadc
1 parent 3d8f3c6
commit 069fadc
Show file tree

Hide file tree

Showing 29 changed files with 543 additions and 46 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -73,6 +73,8 @@ RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends install -yq \
         apache2 \
         libapache2-mod-xsendfile \
+        libgomp1 \
+        libgl1 \
         supervisor \
         libldap-2.4-2 \
         libsasl2-2 \

diff --git a/cvat-core/src/annotation-formats.js b/cvat-core/src/annotation-formats.js
@@ -15,6 +15,7 @@
                 format: initialData.ext,
                 version: initialData.version,
                 enabled: initialData.enabled,
+                dimension: initialData.dimension,
             };
 
             Object.defineProperties(this, {
@@ -58,6 +59,16 @@
                      */
                     get: () => data.enabled,
                 },
+                dimension: {
+                    /**
+                     * @name dimension
+                     * @type {string}
+                     * @memberof module:API.cvat.enums.DimensionType
+                     * @readonly
+                     * @instance
+                     */
+                    get: () => data.dimension,
+                },
             });
         }
     }
@@ -74,6 +85,7 @@
                 format: initialData.ext,
                 version: initialData.version,
                 enabled: initialData.enabled,
+                dimension: initialData.dimension,
             };
 
             Object.defineProperties(this, {
@@ -117,6 +129,16 @@
                      */
                     get: () => data.enabled,
                 },
+                dimension: {
+                    /**
+                     * @name dimension
+                     * @type {string}
+                     * @memberof module:API.cvat.enums.DimensionType
+                     * @readonly
+                     * @instance
+                     */
+                    get: () => data.dimension,
+                },
             });
         }
     }

diff --git a/cvat-core/src/api-implementation.js b/cvat-core/src/api-implementation.js
@@ -10,7 +10,7 @@
         isBoolean, isInteger, isEnum, isString, checkFilter,
     } = require('./common');
 
-    const { TaskStatus, TaskMode } = require('./enums');
+    const { TaskStatus, TaskMode, DimensionType } = require('./enums');
 
     const User = require('./user');
     const { AnnotationFormats } = require('./annotation-formats');
@@ -176,6 +176,7 @@
                 search: isString,
                 status: isEnum.bind(TaskStatus),
                 mode: isEnum.bind(TaskMode),
+                dimension: isEnum.bind(DimensionType),
             });
 
             if ('search' in filter && Object.keys(filter).length > 1) {
@@ -198,7 +199,7 @@
             }
 
             const searchParams = new URLSearchParams();
-            for (const field of ['name', 'owner', 'assignee', 'search', 'status', 'mode', 'id', 'page', 'projectId']) {
+            for (const field of ['name', 'owner', 'assignee', 'search', 'status', 'mode', 'id', 'page', 'projectId', 'dimension']) {
                 if (Object.prototype.hasOwnProperty.call(filter, field)) {
                     searchParams.set(field, filter[field]);
                 }

diff --git a/cvat-core/src/enums.js b/cvat-core/src/enums.js
@@ -33,6 +33,20 @@
         COMPLETED: 'completed',
     });
 
+    /**
+     * Task dimension
+     * @enum
+     * @name DimensionType
+     * @memberof module:API.cvat.enums
+     * @property {string} DIMENSION_2D '2d'
+     * @property {string} DIMENSION_3D '3d'
+     * @readonly
+     */
+    const DimensionType = Object.freeze({
+        DIMENSION_2D: '2d',
+        DIMENSION_3D: '3d',
+    });
+
     /**
      * Review statuses
      * @enum {string}
@@ -333,5 +347,6 @@
         RQStatus,
         colors,
         Source,
+        DimensionType,
     };
 })();
diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js
@@ -974,6 +974,7 @@
                 use_zip_chunks: undefined,
                 use_cache: undefined,
                 copy_data: undefined,
+                dimension: undefined,
             };
 
             let updatedFields = {
@@ -1452,6 +1453,16 @@
                             updatedFields = fields;
                         },
                     },
+                    dimension: {
+                        /**
+                         * @name enabled
+                         * @type {string}
+                         * @memberof module:API.cvat.enums.DimensionType
+                         * @readonly
+                         * @instance
+                         */
+                        get: () => data.dimension,
+                    },
                 }),
             );
 

diff --git a/cvat-ui/src/components/actions-menu/actions-menu.tsx b/cvat-ui/src/components/actions-menu/actions-menu.tsx
@@ -11,6 +11,7 @@ import { MenuInfo } from 'rc-menu/lib/interface';
 import DumpSubmenu from './dump-submenu';
 import LoadSubmenu from './load-submenu';
 import ExportSubmenu from './export-submenu';
+import { DimensionType } from '../../reducers/interfaces';
 
 interface Props {
     taskID: number;
@@ -22,7 +23,7 @@ interface Props {
     dumpActivities: string[] | null;
     exportActivities: string[] | null;
     inferenceIsActive: boolean;
-
+    taskDimension: DimensionType;
     onClickMenu: (params: MenuInfo, file?: File) => void;
 }
 
@@ -47,6 +48,7 @@ export default function ActionsMenuComponent(props: Props): JSX.Element {
         dumpActivities,
         exportActivities,
         loadActivity,
+        taskDimension,
     } = props;
 
     let latestParams: MenuInfo | null = null;
@@ -102,6 +104,7 @@ export default function ActionsMenuComponent(props: Props): JSX.Element {
                 dumpers,
                 dumpActivities,
                 menuKey: Actions.DUMP_TASK_ANNO,
+                taskDimension,
             })}
             {LoadSubmenu({
                 loaders,
@@ -110,11 +113,13 @@ export default function ActionsMenuComponent(props: Props): JSX.Element {
                     onClickMenuWrapper(null, file);
                 },
                 menuKey: Actions.LOAD_TASK_ANNO,
+                taskDimension,
             })}
             {ExportSubmenu({
                 exporters: dumpers,
                 exportActivities,
                 menuKey: Actions.EXPORT_TASK_DATASET,
+                taskDimension,
             })}
             {!!bugTracker && <Menu.Item key={Actions.OPEN_BUG_TRACKER}>Open bug tracker</Menu.Item>}
             <Menu.Item disabled={inferenceIsActive} key={Actions.RUN_AUTO_ANNOTATION}>

diff --git a/cvat-ui/src/components/actions-menu/dump-submenu.tsx b/cvat-ui/src/components/actions-menu/dump-submenu.tsx
@@ -6,6 +6,7 @@ import React from 'react';
 import Menu from 'antd/lib/menu';
 import { DownloadOutlined, LoadingOutlined } from '@ant-design/icons';
 import Text from 'antd/lib/typography/Text';
+import { DimensionType } from '../../reducers/interfaces';
 
 function isDefaultFormat(dumperName: string, taskMode: string): boolean {
     return (
@@ -19,15 +20,19 @@ interface Props {
     menuKey: string;
     dumpers: any[];
     dumpActivities: string[] | null;
+    taskDimension: DimensionType;
 }
 
 export default function DumpSubmenu(props: Props): JSX.Element {
-    const { taskMode, menuKey, dumpers, dumpActivities } = props;
+    const {
+        taskMode, menuKey, dumpers, dumpActivities, taskDimension,
+    } = props;
 
     return (
         <Menu.SubMenu key={menuKey} title='Dump annotations'>
             {dumpers
                 .sort((a: any, b: any) => a.name.localeCompare(b.name))
+                .filter((dumper: any): boolean => dumper.dimension === taskDimension)
                 .map(
                     (dumper: any): JSX.Element => {
                         const pending = (dumpActivities || []).includes(dumper.name);

diff --git a/cvat-ui/src/components/actions-menu/export-submenu.tsx b/cvat-ui/src/components/actions-menu/export-submenu.tsx
@@ -6,20 +6,25 @@ import React from 'react';
 import Menu from 'antd/lib/menu';
 import Text from 'antd/lib/typography/Text';
 import { ExportOutlined, LoadingOutlined } from '@ant-design/icons';
+import { DimensionType } from '../../reducers/interfaces';
 
 interface Props {
     menuKey: string;
     exporters: any[];
     exportActivities: string[] | null;
+    taskDimension: DimensionType;
 }
 
 export default function ExportSubmenu(props: Props): JSX.Element {
-    const { menuKey, exporters, exportActivities } = props;
+    const {
+        menuKey, exporters, exportActivities, taskDimension,
+    } = props;
 
     return (
         <Menu.SubMenu key={menuKey} title='Export as a dataset'>
             {exporters
                 .sort((a: any, b: any) => a.name.localeCompare(b.name))
+                .filter((exporter: any): boolean => exporter.dimension === taskDimension)
                 .map(
                     (exporter: any): JSX.Element => {
                         const pending = (exportActivities || []).includes(exporter.name);

diff --git a/cvat-ui/src/components/actions-menu/load-submenu.tsx b/cvat-ui/src/components/actions-menu/load-submenu.tsx
@@ -8,23 +8,26 @@ import Upload from 'antd/lib/upload';
 import Button from 'antd/lib/button';
 import Text from 'antd/lib/typography/Text';
 import { UploadOutlined, LoadingOutlined } from '@ant-design/icons';
+import { DimensionType } from '../../reducers/interfaces';
 
 interface Props {
     menuKey: string;
     loaders: any[];
     loadActivity: string | null;
     onFileUpload(file: File): void;
+    taskDimension: DimensionType;
 }
 
 export default function LoadSubmenu(props: Props): JSX.Element {
     const {
-        menuKey, loaders, loadActivity, onFileUpload,
+        menuKey, loaders, loadActivity, onFileUpload, taskDimension,
     } = props;
 
     return (
         <Menu.SubMenu key={menuKey} title='Upload annotations'>
             {loaders
                 .sort((a: any, b: any) => a.name.localeCompare(b.name))
+                .filter((loader: any): boolean => loader.dimension === taskDimension)
                 .map(
                     (loader: any): JSX.Element => {
                         const accept = loader.format

diff --git a/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx b/cvat-ui/src/components/annotation-page/top-bar/annotation-menu.tsx
@@ -164,6 +164,7 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element {
                 dumpers,
                 dumpActivities,
                 menuKey: Actions.DUMP_TASK_ANNO,
+                taskDimension: jobInstance.task.dimension,
             })}
             {LoadSubmenu({
                 loaders,
@@ -172,11 +173,13 @@ export default function AnnotationMenuComponent(props: Props): JSX.Element {
                     onClickMenuWrapper(null, file);
                 },
                 menuKey: Actions.LOAD_JOB_ANNO,
+                taskDimension: jobInstance.task.dimension,
             })}
             {ExportSubmenu({
                 exporters: dumpers,
                 exportActivities,
                 menuKey: Actions.EXPORT_TASK_DATASET,
+                taskDimension: jobInstance.task.dimension,
             })}
 
             <Menu.Item key={Actions.REMOVE_ANNO}>Remove annotations</Menu.Item>

diff --git a/cvat-ui/src/components/model-runner-modal/detector-runner.tsx b/cvat-ui/src/components/model-runner-modal/detector-runner.tsx
@@ -21,6 +21,7 @@ import { Model, StringObject } from 'reducers/interfaces';
 
 import { clamp } from 'utils/math';
 import consts from 'consts';
+import { DimensionType } from '../../reducers/interfaces';
 
 interface Props {
     withCleanup: boolean;
@@ -127,7 +128,8 @@ function DetectorRunner(props: Props): JSX.Element {
                 <Col span={4}>Model:</Col>
                 <Col span={20}>
                     <Select
-                        placeholder='Select a model'
+                        placeholder={task.dimension === DimensionType.DIM_2D ? 'Select a model' : 'No models available'}
+                        disabled={task.dimension !== DimensionType.DIM_2D}
                         style={{ width: '100%' }}
                         onChange={(_modelID: string): void => {
                             const newmodel = models.filter((_model): boolean => _model.id === _modelID)[0];

diff --git a/cvat-ui/src/containers/actions-menu/actions-menu.tsx b/cvat-ui/src/containers/actions-menu/actions-menu.tsx
@@ -137,6 +137,7 @@ function ActionsMenuContainer(props: OwnProps & StateToProps & DispatchToProps):
             exportActivities={exportActivities}
             inferenceIsActive={inferenceIsActive}
             onClickMenu={onClickMenu}
+            taskDimension={taskInstance.dimension}
         />
     );
 }

diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts
@@ -568,3 +568,8 @@ export interface CombinedState {
     shortcuts: ShortcutsState;
     review: ReviewState;
 }
+
+export enum DimensionType {
+    DIM_3D = '3d',
+    DIM_2D = '2d',
+}
diff --git a/cvat/apps/dataset_manager/formats/registry.py b/cvat/apps/dataset_manager/formats/registry.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: MIT
 
 from datumaro.components.project import Environment
+from cvat.apps.engine.models import DimensionType
 
 
 dm_env = Environment()
@@ -23,7 +24,7 @@ class Importer(_Format):
     def __call__(self, src_file, task_data, **options):
         raise NotImplementedError()
 
-def _wrap_format(f_or_cls, klass, name, version, ext, display_name, enabled):
+def _wrap_format(f_or_cls, klass, name, version, ext, display_name, enabled, dimension=DimensionType.DIM_2D):
     import inspect
     assert inspect.isclass(f_or_cls) or inspect.isfunction(f_or_cls)
     if inspect.isclass(f_or_cls):
@@ -45,29 +46,30 @@ def __call__(self, *args, **kwargs):
     target.DISPLAY_NAME = (display_name or klass.DISPLAY_NAME).format(
         NAME=name, VERSION=version, EXT=ext)
     assert all([target.NAME, target.VERSION, target.EXT, target.DISPLAY_NAME])
+    target.DIMENSION = dimension
     target.ENABLED = enabled
 
     return target
 
 EXPORT_FORMATS = {}
-def exporter(name, version, ext, display_name=None, enabled=True):
+def exporter(name, version, ext, display_name=None, enabled=True, dimension=DimensionType.DIM_2D):
     assert name not in EXPORT_FORMATS, "Export format '%s' already registered" % name
     def wrap_with_params(f_or_cls):
         t = _wrap_format(f_or_cls, Exporter,
             name=name, ext=ext, version=version, display_name=display_name,
-            enabled=enabled)
+            enabled=enabled, dimension=dimension)
         key = t.DISPLAY_NAME
         assert key not in EXPORT_FORMATS, "Export format '%s' already registered" % name
         EXPORT_FORMATS[key] = t
         return t
     return wrap_with_params
 
 IMPORT_FORMATS = {}
-def importer(name, version, ext, display_name=None, enabled=True):
+def importer(name, version, ext, display_name=None, enabled=True, dimension=DimensionType.DIM_2D):
     def wrap_with_params(f_or_cls):
         t = _wrap_format(f_or_cls, Importer,
             name=name, ext=ext, version=version, display_name=display_name,
-            enabled=enabled)
+            enabled=enabled, dimension=dimension)
         key = t.DISPLAY_NAME
         assert key not in IMPORT_FORMATS, "Import format '%s' already registered" % name
         IMPORT_FORMATS[key] = t
@@ -92,4 +94,4 @@ def make_exporter(name):
 import cvat.apps.dataset_manager.formats.tfrecord
 import cvat.apps.dataset_manager.formats.yolo
 import cvat.apps.dataset_manager.formats.imagenet
-import cvat.apps.dataset_manager.formats.camvid
+import cvat.apps.dataset_manager.formats.camvid
diff --git a/cvat/apps/dataset_manager/serializers.py b/cvat/apps/dataset_manager/serializers.py
@@ -10,7 +10,8 @@ class DatasetFormatSerializer(serializers.Serializer):
     ext = serializers.CharField(max_length=64, source='EXT')
     version = serializers.CharField(max_length=64, source='VERSION')
     enabled = serializers.BooleanField(source='ENABLED')
+    dimension = serializers.CharField(max_length=2, source='DIMENSION')
 
 class DatasetFormatsSerializer(serializers.Serializer):
     importers = DatasetFormatSerializer(many=True)
-    exporters = DatasetFormatSerializer(many=True)
+    exporters = DatasetFormatSerializer(many=True)
diff --git a/cvat/apps/engine/assets/3d_preview.jpeg b/cvat/apps/engine/assets/3d_preview.jpeg