[Datumaro] CVAT format import

datumaro/datumaro/cli/project/__init__.py

@@ -7,6 +7,7 @@
 import logging as log
 import os
 import os.path as osp
+import shutil
 
 from datumaro.components.project import Project
 from datumaro.components.comparator import Comparator
@@ -27,10 +28,13 @@ def create_command(args):
     project_dir = osp.abspath(args.dst_dir)
     project_path = make_project_path(project_dir)
 
-    if not args.overwrite and osp.isdir(project_dir) and os.listdir(project_dir):
-        log.error("Directory '%s' already exists "
-            "(pass --overwrite to force creation)" % project_dir)
-        return 1
+    if osp.isdir(project_dir) and os.listdir(project_dir):
+        if not args.overwrite:
+            log.error("Directory '%s' already exists "
+                "(pass --overwrite to force creation)" % project_dir)
+            return 1
+        else:
+            shutil.rmtree(project_dir)
     os.makedirs(project_dir, exist_ok=args.overwrite)
 
     if not args.overwrite and osp.isfile(project_path):
@@ -76,10 +80,13 @@ def import_command(args):
     project_dir = osp.abspath(args.dst_dir)
     project_path = make_project_path(project_dir)
 
-    if not args.overwrite and osp.isdir(project_dir) and os.listdir(project_dir):
-        log.error("Directory '%s' already exists "
-            "(pass --overwrite to force creation)" % project_dir)
-        return 1
+    if osp.isdir(project_dir) and os.listdir(project_dir):
+        if not args.overwrite:
+            log.error("Directory '%s' already exists "
+                "(pass --overwrite to force creation)" % project_dir)
+            return 1
+        else:
+            shutil.rmtree(project_dir)
     os.makedirs(project_dir, exist_ok=args.overwrite)
 
     if not args.overwrite and osp.isfile(project_path):
@@ -137,7 +144,11 @@ def export_command(args):
     dst_dir = osp.abspath(args.dst_dir)
     os.makedirs(dst_dir, exist_ok=False)
 
-    project.make_dataset().export(
+    log.info("Loading the project...")
+    dataset = project.make_dataset()
+
+    log.info("Exporting the project...")
+    dataset.export(
         save_dir=dst_dir,
         output_format=args.output_format,
         filter_expr=args.filter,

datumaro/datumaro/cli/source/__init__.py

@@ -194,8 +194,12 @@ def export_command(args):
     dst_dir = osp.abspath(args.dst_dir)
     os.makedirs(dst_dir, exist_ok=False)
 
+    log.info("Loading the project...")
     source_project = project.make_source_project(args.name)
-    source_project.make_dataset().export(
+    dataset = source_project.make_dataset()
+
+    log.info("Exporting the project...")
+    dataset.export(
         save_dir=dst_dir,
         output_format=args.output_format,
         filter_expr=args.filter,

datumaro/datumaro/components/converters/__init__.py

@@ -24,10 +24,7 @@
 )
 
 from datumaro.components.converters.yolo import YoloConverter
-
-from datumaro.components.converters.tfrecord import (
-    DetectionApiConverter,
-)
+from datumaro.components.converters.tfrecord import DetectionApiConverter
 
 
 items = [

datumaro/datumaro/components/converters/datumaro.py

@@ -11,8 +11,7 @@
 
 from datumaro.components.converter import Converter
 from datumaro.components.extractor import (
-    DEFAULT_SUBSET_NAME,
-    AnnotationType, Annotation,
+    DEFAULT_SUBSET_NAME, Annotation,
     LabelObject, MaskObject, PointsObject, PolygonObject,
     PolyLineObject, BboxObject, CaptionObject,
     LabelCategories, MaskCategories, PointsCategories
@@ -52,11 +51,13 @@ def items(self):
 
     def write_item(self, item):
         annotations = []
-        self.items.append({
+        item_desc = {
             'id': item.id,
-            'path': item.path,
             'annotations': annotations,
-        })
+        }
+        if item.path:
+            item_desc['path'] = item.path
+        self.items.append(item_desc)
 
         for ann in item.annotations:
             if isinstance(ann, LabelObject):

datumaro/datumaro/components/extractor.py

@@ -53,6 +53,11 @@ class Categories:
     def __init__(self, attributes=None):
         if attributes is None:
             attributes = set()
+        else:
+            if not isinstance(attributes, set):
+                attributes = set(attributes)
+            for attr in attributes:
+                assert isinstance(attr, str)
         self.attributes = attributes
 
     def __eq__(self, other):
@@ -62,7 +67,7 @@ def __eq__(self, other):
             (self.attributes == other.attributes)
 
 class LabelCategories(Categories):
-    Category = namedtuple('Category', ['name', 'parent'])
+    Category = namedtuple('Category', ['name', 'parent', 'attributes'])
 
     def __init__(self, items=None, attributes=None):
         super().__init__(attributes=attributes)
@@ -81,11 +86,18 @@ def _reindex(self):
             indices[item.name] = index
         self._indices = indices
 
-    def add(self, name, parent=None):
+    def add(self, name, parent=None, attributes=None):
         assert name not in self._indices
+        if attributes is None:
+            attributes = set()
+        else:
+            if not isinstance(attributes, set):
+                attributes = set(attributes)
+            for attr in attributes:
+                assert isinstance(attr, str)
 
         index = len(self.items)
-        self.items.append(self.Category(name, parent))
+        self.items.append(self.Category(name, parent, attributes))
         self._indices[name] = index
 
     def find(self, name):
@@ -462,6 +474,7 @@ def __eq__(self, other):
             (self.id == other.id) and \
             (self.subset == other.subset) and \
             (self.annotations == other.annotations) and \
+            (self.path == other.path) and \
             (self.has_image == other.has_image) and \
             (self.has_image and np.all(self.image == other.image) or \
                 not self.has_image)

datumaro/datumaro/components/extractors/__init__.py

@@ -26,14 +26,9 @@
     VocComp_9_10_Extractor,
 )
 
-from datumaro.components.extractors.yolo import (
-    YoloExtractor,
-)
-
-from datumaro.components.extractors.tfrecord import (
-    DetectionApiExtractor,
-)
-
+from datumaro.components.extractors.yolo import YoloExtractor
+from datumaro.components.extractors.tfrecord import DetectionApiExtractor
+from datumaro.components.extractors.cvat import CvatExtractor
 
 items = [
     ('datumaro', DatumaroExtractor),
@@ -59,4 +54,6 @@
     ('yolo', YoloExtractor),
 
     ('tf_detection_api', DetectionApiExtractor),
+
+    ('cvat', CvatExtractor),
 ]