From 7772613b3e641e0e47e93a947e21818499b6b32e Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 11 Jan 2021 14:30:17 +0300 Subject: [PATCH] Update dataset importing (#79) --- datumaro/util/test_utils.py | 5 ++--- tests/test_camvid_format.py | 8 ++++---- tests/test_coco_format.py | 5 ++--- tests/test_dataset.py | 4 +--- tests/test_imagenet_format.py | 7 +++---- tests/test_imagenet_txt_format.py | 8 ++++---- tests/test_labelme_format.py | 6 ++---- tests/test_mot_format.py | 6 ++---- tests/test_mots_format.py | 4 ++-- tests/test_tfrecord_format.py | 6 ++---- tests/test_vgg_face2_format.py | 5 ++--- tests/test_voc_format.py | 4 ++-- tests/test_widerface_format.py | 5 ++--- tests/test_yolo_format.py | 5 ++--- 14 files changed, 32 insertions(+), 46 deletions(-) diff --git a/datumaro/util/test_utils.py b/datumaro/util/test_utils.py index 08ffa4ed53af..63bd4222c040 100644 --- a/datumaro/util/test_utils.py +++ b/datumaro/util/test_utils.py @@ -10,7 +10,7 @@ import tempfile from datumaro.components.extractor import AnnotationType -from datumaro.components.project import Project +from datumaro.components.dataset import Dataset from datumaro.util import find @@ -132,8 +132,7 @@ def test_save_and_load(test, source_dataset, converter, test_dir, importer, if importer_args is None: importer_args = {} - parsed_dataset = Project.import_from(test_dir, importer, **importer_args) \ - .make_dataset() + parsed_dataset = Dataset.import_from(test_dir, importer, **importer_args) if target_dataset is None: target_dataset = source_dataset diff --git a/tests/test_camvid_format.py b/tests/test_camvid_format.py index 12192279b915..85e0b6e7d97e 100644 --- a/tests/test_camvid_format.py +++ b/tests/test_camvid_format.py @@ -6,11 +6,11 @@ import datumaro.plugins.camvid_format as Camvid import numpy as np from datumaro.components.extractor import (AnnotationType, DatasetItem, - Extractor, LabelCategories, Mask) -from datumaro.components.project import Dataset, Project + Extractor, LabelCategories, Mask) +from datumaro.components.dataset import Dataset from datumaro.plugins.camvid_format import CamvidConverter, CamvidImporter from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) + test_save_and_load) class CamvidFormatTest(TestCase): @@ -68,7 +68,7 @@ def test_can_import(self): ), ], categories=Camvid.make_camvid_categories()) - parsed_dataset = Project.import_from(DUMMY_DATASET_DIR, 'camvid').make_dataset() + parsed_dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'camvid') compare_datasets(self, source_dataset, parsed_dataset) diff --git a/tests/test_coco_format.py b/tests/test_coco_format.py index a25f15f56160..f97d9255f88e 100644 --- a/tests/test_coco_format.py +++ b/tests/test_coco_format.py @@ -4,7 +4,7 @@ from unittest import TestCase -from datumaro.components.project import Project, Dataset +from datumaro.components.dataset import Dataset from datumaro.components.extractor import (DatasetItem, AnnotationType, Label, Mask, Points, Polygon, Bbox, Caption, LabelCategories, PointsCategories @@ -42,8 +42,7 @@ def test_can_import(self): ), ], categories=['TEST',]) - dataset = Project.import_from(DUMMY_DATASET_DIR, 'coco') \ - .make_dataset() + dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'coco') compare_datasets(self, expected_dataset, dataset) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index f7613c4e1a7f..d2e5e4b3c8f1 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -2,7 +2,7 @@ from unittest import TestCase -from datumaro.components.project import Environment +from datumaro.components.environment import Environment from datumaro.components.extractor import (Extractor, DatasetItem, Label, Mask, Points, Polygon, PolyLine, Bbox, Caption, LabelCategories, AnnotationType, Transform @@ -156,8 +156,6 @@ def transform_item(self, item): actual = dataset.transform('qq') - self.assertTrue(isinstance(actual, Dataset)) - self.assertEqual(env, actual.env) compare_datasets(self, expected, actual) def test_can_join_annotations(self): diff --git a/tests/test_imagenet_format.py b/tests/test_imagenet_format.py index b6bd4e0c9766..779e46b3100d 100644 --- a/tests/test_imagenet_format.py +++ b/tests/test_imagenet_format.py @@ -3,12 +3,11 @@ import numpy as np import os.path as osp -from datumaro.components.project import Project, Dataset +from datumaro.components.dataset import Dataset from datumaro.components.extractor import (DatasetItem, Label, LabelCategories, AnnotationType ) -from datumaro.plugins.imagenet_format import ImagenetConverter -from datumaro.plugins.imagenet_format import ImagenetImporter +from datumaro.plugins.imagenet_format import ImagenetConverter, ImagenetImporter from datumaro.util.test_utils import TestDir, compare_datasets class ImagenetFormatTest(TestCase): @@ -103,7 +102,7 @@ def test_can_import(self): 'label_' + str(label) for label in range(2)), }) - dataset = Project.import_from(DUMMY_DATASET_DIR, 'imagenet').make_dataset() + dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'imagenet') compare_datasets(self, expected_dataset, dataset, require_images=True) diff --git a/tests/test_imagenet_txt_format.py b/tests/test_imagenet_txt_format.py index 251c71fcc91f..1cd515992308 100644 --- a/tests/test_imagenet_txt_format.py +++ b/tests/test_imagenet_txt_format.py @@ -3,11 +3,12 @@ import numpy as np import os.path as osp -from datumaro.components.project import Project, Dataset +from datumaro.components.dataset import Dataset from datumaro.components.extractor import (DatasetItem, Label, LabelCategories, AnnotationType ) -from datumaro.plugins.imagenet_txt_format import ImagenetTxtConverter, ImagenetTxtImporter +from datumaro.plugins.imagenet_txt_format import \ + ImagenetTxtConverter, ImagenetTxtImporter from datumaro.util.test_utils import TestDir, compare_datasets @@ -111,8 +112,7 @@ def test_can_import(self): 'label_%s' % label for label in range(10)), }) - dataset = Project.import_from(DUMMY_DATASET_DIR, 'imagenet_txt') \ - .make_dataset() + dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'imagenet_txt') compare_datasets(self, expected_dataset, dataset, require_images=True) diff --git a/tests/test_labelme_format.py b/tests/test_labelme_format.py index f51922224f92..244a590b07f4 100644 --- a/tests/test_labelme_format.py +++ b/tests/test_labelme_format.py @@ -3,11 +3,10 @@ import os.path as osp from unittest import TestCase -from datumaro.components.project import Dataset +from datumaro.components.dataset import Dataset from datumaro.components.extractor import (DatasetItem, AnnotationType, Bbox, Mask, Polygon, LabelCategories ) -from datumaro.components.project import Project from datumaro.plugins.labelme_format import LabelMeImporter, LabelMeConverter from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) @@ -181,6 +180,5 @@ def test_can_import(self): ]), }) - parsed = Project.import_from(DUMMY_DATASET_DIR, 'label_me') \ - .make_dataset() + parsed = Dataset.import_from(DUMMY_DATASET_DIR, 'label_me') compare_datasets(self, expected=target_dataset, actual=parsed) \ No newline at end of file diff --git a/tests/test_mot_format.py b/tests/test_mot_format.py index fd647426bbfe..259bde298ad0 100644 --- a/tests/test_mot_format.py +++ b/tests/test_mot_format.py @@ -3,11 +3,10 @@ import os.path as osp from unittest import TestCase -from datumaro.components.project import Dataset +from datumaro.components.dataset import Dataset from datumaro.components.extractor import (DatasetItem, AnnotationType, Bbox, LabelCategories ) -from datumaro.components.project import Project from datumaro.plugins.mot_format import MotSeqGtConverter, MotSeqImporter from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) @@ -123,7 +122,6 @@ def test_can_import(self): 'label_' + str(label) for label in range(10)), }) - dataset = Project.import_from(DUMMY_DATASET_DIR, 'mot_seq') \ - .make_dataset() + dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'mot_seq') compare_datasets(self, expected_dataset, dataset) \ No newline at end of file diff --git a/tests/test_mots_format.py b/tests/test_mots_format.py index 2a6ec057e4ed..f8358dda3cb8 100644 --- a/tests/test_mots_format.py +++ b/tests/test_mots_format.py @@ -5,7 +5,7 @@ from unittest import TestCase from datumaro.components.extractor import DatasetItem, Mask -from datumaro.components.project import Dataset, Project +from datumaro.components.dataset import Dataset from datumaro.plugins.mots_format import MotsPngConverter, MotsImporter from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) @@ -90,5 +90,5 @@ def test_can_import(self): ]), ], categories=['a', 'b', 'c', 'd']) - parsed = Project.import_from(DUMMY_DATASET_DIR, 'mots').make_dataset() + parsed = Dataset.import_from(DUMMY_DATASET_DIR, 'mots') compare_datasets(self, expected=target, actual=parsed) \ No newline at end of file diff --git a/tests/test_tfrecord_format.py b/tests/test_tfrecord_format.py index f9491c39effa..39cbe7c9b498 100644 --- a/tests/test_tfrecord_format.py +++ b/tests/test_tfrecord_format.py @@ -4,11 +4,10 @@ from unittest import TestCase, skipIf -from datumaro.components.project import Dataset +from datumaro.components.dataset import Dataset from datumaro.components.extractor import (DatasetItem, AnnotationType, Bbox, Mask, LabelCategories ) -from datumaro.components.project import Project from datumaro.util.image import Image, ByteImage, encode_image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) @@ -218,7 +217,6 @@ def test_can_import(self): 'label_' + str(label) for label in range(10)), }) - dataset = Project.import_from(DUMMY_DATASET_DIR, 'tf_detection_api') \ - .make_dataset() + dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'tf_detection_api') compare_datasets(self, target_dataset, dataset) diff --git a/tests/test_vgg_face2_format.py b/tests/test_vgg_face2_format.py index 5c3bc02465a5..71fd1d322250 100644 --- a/tests/test_vgg_face2_format.py +++ b/tests/test_vgg_face2_format.py @@ -3,7 +3,7 @@ import numpy as np from datumaro.components.extractor import Bbox, DatasetItem, Points -from datumaro.components.project import Dataset, Project +from datumaro.components.dataset import Dataset from datumaro.plugins.vgg_face2_format import (VggFace2Converter, VggFace2Importer) from datumaro.util.test_utils import TestDir, compare_datasets @@ -93,7 +93,6 @@ def test_can_import(self): ), ], categories=[]) - dataset = Project.import_from(DUMMY_DATASET_DIR, 'vgg_face2') \ - .make_dataset() + dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'vgg_face2') compare_datasets(self, expected_dataset, dataset) diff --git a/tests/test_voc_format.py b/tests/test_voc_format.py index b33aaa125f51..9a4502f6e91b 100644 --- a/tests/test_voc_format.py +++ b/tests/test_voc_format.py @@ -18,7 +18,7 @@ VocSegmentationConverter, ) from datumaro.plugins.voc_format.importer import VocImporter -from datumaro.components.project import Project +from datumaro.components.dataset import Dataset from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) @@ -122,7 +122,7 @@ def __iter__(self): image=np.ones((10, 20, 3))), ]) - dataset = Project.import_from(DUMMY_DATASET_DIR, 'voc').make_dataset() + dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'voc') compare_datasets(self, DstExtractor(), dataset) diff --git a/tests/test_widerface_format.py b/tests/test_widerface_format.py index b3af94dda1cb..ed51379ba6bd 100644 --- a/tests/test_widerface_format.py +++ b/tests/test_widerface_format.py @@ -3,7 +3,7 @@ import numpy as np from datumaro.components.extractor import Bbox, DatasetItem -from datumaro.components.project import Dataset, Project +from datumaro.components.dataset import Dataset from datumaro.plugins.widerface_format import WiderFaceConverter, WiderFaceImporter from datumaro.util.test_utils import TestDir, compare_datasets @@ -147,7 +147,6 @@ def test_can_import(self): ), ]) - dataset = Project.import_from(DUMMY_DATASET_DIR, 'wider_face') \ - .make_dataset() + dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'wider_face') compare_datasets(self, expected_dataset, dataset) diff --git a/tests/test_yolo_format.py b/tests/test_yolo_format.py index 549811fd1ce0..615bb6d19f39 100644 --- a/tests/test_yolo_format.py +++ b/tests/test_yolo_format.py @@ -6,7 +6,7 @@ from datumaro.components.extractor import (DatasetItem, AnnotationType, Bbox, LabelCategories, ) -from datumaro.components.project import Project, Dataset +from datumaro.components.dataset import Dataset from datumaro.plugins.yolo_format.extractor import YoloImporter from datumaro.plugins.yolo_format.converter import YoloConverter from datumaro.util.image import Image, save_image @@ -130,7 +130,6 @@ def test_can_import(self): 'label_' + str(i) for i in range(10)), }) - dataset = Project.import_from(DUMMY_DATASET_DIR, 'yolo') \ - .make_dataset() + dataset = Dataset.import_from(DUMMY_DATASET_DIR, 'yolo') compare_datasets(self, expected_dataset, dataset)