From 12e7fc62af1e327483a60af4cb96094213d3c98a Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Tue, 29 Jun 2021 20:07:05 +0300 Subject: [PATCH 1/3] Add an isort config Disable Pylint warnings regarding import order, since isort will be managing that from now on. --- .isort.cfg | 6 ++++++ .pylintrc | 2 +- datumaro/plugins/accuracy_checker_plugin/details/ac.py | 2 +- .../accuracy_checker_plugin/details/representation.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000000..ef57375c8b --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,6 @@ +[settings] +forced_separate = tests +from_first = True +include_trailing_comma = True +line_length = 80 +multi_line_output = 5 diff --git a/.pylintrc b/.pylintrc index 2c9edce81b..92f0a4f3a9 100644 --- a/.pylintrc +++ b/.pylintrc @@ -61,7 +61,7 @@ enable= E0001,E0100,E0101,E0102,E0103,E0104,E0105,E0106,E0107,E0110, E0702,E0703,E0704,E0710,E0711,E0712,E1003,E1102,E1111,E0112, E1120,E1121,E1123,E1124,E1125,E1126,E1127,E1132,E1200,E1201, E1205,E1206,E1300,E1301,E1302,E1303,E1304,E1305,E1306, - C0123,C0200,C0303,C0411,C1001, + C0123,C0200,C0303,C1001, W0101,W0102,W0104,W0105,W0106,W0107,W0108,W0109,W0110,W0120, W0122,W0124,W0150,W0199,W0221,W0222,W0233,W0404,W0410,W0601, W0602,W0604,W0611,W0612,W0622,W0623,W0702,W0705,W0711,W1300, diff --git a/datumaro/plugins/accuracy_checker_plugin/details/ac.py b/datumaro/plugins/accuracy_checker_plugin/details/ac.py index 2a60f41a7f..b235e57848 100644 --- a/datumaro/plugins/accuracy_checker_plugin/details/ac.py +++ b/datumaro/plugins/accuracy_checker_plugin/details/ac.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: MIT -from datumaro.util.tf_util import import_tf # pylint: disable=wrong-import-order +from datumaro.util.tf_util import import_tf import_tf() # prevent TF loading and potential interpreter crash from itertools import groupby diff --git a/datumaro/plugins/accuracy_checker_plugin/details/representation.py b/datumaro/plugins/accuracy_checker_plugin/details/representation.py index 5324710188..d502dd2b3a 100644 --- a/datumaro/plugins/accuracy_checker_plugin/details/representation.py +++ b/datumaro/plugins/accuracy_checker_plugin/details/representation.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: MIT -from datumaro.util.tf_util import import_tf # pylint: disable=wrong-import-order +from datumaro.util.tf_util import import_tf import_tf() # prevent TF loading and potential interpreter crash import accuracy_checker.representation as ac From ae4d95875c7c1ade375562767780bf1bd40d5fe0 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Tue, 6 Jul 2021 13:06:27 +0300 Subject: [PATCH 2/3] Reformat all imports using isort --- datum.py | 1 - datumaro/__main__.py | 1 - datumaro/cli/__main__.py | 5 ++- datumaro/cli/commands/__init__.py | 7 ++-- datumaro/cli/commands/convert.py | 2 +- datumaro/cli/commands/explain.py | 9 +++-- datumaro/cli/commands/merge.py | 8 ++--- datumaro/cli/contexts/__init__.py | 2 +- datumaro/cli/contexts/model.py | 5 +-- datumaro/cli/contexts/project/__init__.py | 17 ++++++---- datumaro/cli/contexts/project/diff.py | 3 +- datumaro/cli/contexts/source.py | 3 +- datumaro/cli/util/__init__.py | 3 +- datumaro/components/algorithms/rise.py | 1 + datumaro/components/cli_plugin.py | 7 ++-- datumaro/components/config_model.py | 7 ++-- datumaro/components/converter.py | 2 +- datumaro/components/dataset.py | 24 +++++++------ datumaro/components/dataset_filter.py | 10 +++--- datumaro/components/environment.py | 16 +++++---- datumaro/components/extractor.py | 8 ++--- datumaro/components/launcher.py | 5 +-- datumaro/components/operations.py | 29 +++++++++------- datumaro/components/project.py | 12 ++++--- .../accuracy_checker_plugin/details/ac.py | 1 + .../details/representation.py | 4 ++- .../accuracy_checker_plugin/launcher.py | 1 + datumaro/plugins/camvid_format.py | 11 +++--- datumaro/plugins/cifar_format.py | 9 +++-- datumaro/plugins/cityscapes_format.py | 16 ++++----- datumaro/plugins/coco_format/converter.py | 16 +++++---- datumaro/plugins/coco_format/extractor.py | 10 +++--- datumaro/plugins/coco_format/importer.py | 2 +- datumaro/plugins/cvat_format/converter.py | 11 +++--- datumaro/plugins/cvat_format/extractor.py | 7 ++-- datumaro/plugins/datumaro_format/converter.py | 9 ++--- datumaro/plugins/datumaro_format/extractor.py | 7 ++-- datumaro/plugins/icdar_format/extractor.py | 6 ++-- datumaro/plugins/image_dir_format.py | 2 +- datumaro/plugins/image_zip_format.py | 4 +-- datumaro/plugins/imagenet_format.py | 7 ++-- datumaro/plugins/imagenet_txt_format.py | 7 ++-- datumaro/plugins/kitti_format/converter.py | 18 +++++----- datumaro/plugins/kitti_format/extractor.py | 12 ++++--- datumaro/plugins/kitti_format/format.py | 5 ++- datumaro/plugins/kitti_format/importer.py | 5 +-- datumaro/plugins/labelme_format.py | 11 +++--- datumaro/plugins/lfw_format.py | 6 ++-- datumaro/plugins/market1501_format.py | 7 ++-- datumaro/plugins/mnist_csv_format.py | 7 ++-- datumaro/plugins/mnist_format.py | 7 ++-- datumaro/plugins/mot_format.py | 10 +++--- datumaro/plugins/mots_format.py | 8 +++-- datumaro/plugins/ndr.py | 4 +-- datumaro/plugins/open_images_format.py | 6 ++-- datumaro/plugins/openvino_plugin/launcher.py | 6 ++-- .../samples/mobilenet_v2_pytorch_interp.py | 2 +- datumaro/plugins/sampler/sampler.py | 4 +-- datumaro/plugins/splitter.py | 12 ++++--- .../tf_detection_api_format/converter.py | 12 ++++--- .../tf_detection_api_format/extractor.py | 7 ++-- datumaro/plugins/transforms.py | 13 +++---- datumaro/plugins/validators.py | 18 +++++----- datumaro/plugins/vgg_face2_format.py | 6 ++-- datumaro/plugins/voc_format/converter.py | 16 +++++---- datumaro/plugins/voc_format/extractor.py | 15 ++++---- datumaro/plugins/voc_format/format.py | 5 +-- datumaro/plugins/voc_format/importer.py | 4 +-- datumaro/plugins/widerface_format.py | 6 ++-- datumaro/plugins/yolo_format/converter.py | 7 ++-- datumaro/plugins/yolo_format/extractor.py | 7 ++-- datumaro/util/annotation_util.py | 5 +-- datumaro/util/command_targets.py | 2 +- datumaro/util/image.py | 6 ++-- datumaro/util/image_cache.py | 1 - datumaro/util/log_utils.py | 1 + datumaro/util/mask_tools.py | 1 + datumaro/util/os_util.py | 1 - datumaro/util/test_utils.py | 4 +-- datumaro/util/tf_util.py | 2 +- setup.py | 1 + tests/cli/test_diff.py | 11 +++--- tests/cli/test_image_zip_format.py | 5 +-- tests/cli/test_voc_format.py | 8 +++-- tests/cli/test_yolo_format.py | 8 +++-- tests/requirements.py | 2 +- tests/test_RISE.py | 6 ++-- tests/test_camvid_format.py | 16 +++++---- tests/test_cifar_format.py | 9 +++-- tests/test_cityscapes_format.py | 21 +++++++----- tests/test_coco_format.py | 34 +++++++++---------- tests/test_command_targets.py | 9 +++-- tests/test_config.py | 1 + tests/test_cvat_format.py | 19 ++++++----- tests/test_dataset.py | 19 +++++++---- tests/test_datumaro_format.py | 22 ++++++------ tests/test_diff.py | 8 +++-- tests/test_icdar_format.py | 21 +++++++----- tests/test_image.py | 6 ++-- tests/test_image_dir_format.py | 10 ++++-- tests/test_image_zip_format.py | 5 +-- tests/test_imagenet_format.py | 8 +++-- tests/test_imagenet_txt_format.py | 13 ++++--- tests/test_images.py | 11 +++--- tests/test_kitti_format.py | 25 +++++++------- tests/test_labelme_format.py | 13 ++++--- tests/test_lfw_format.py | 4 ++- tests/test_market1501_format.py | 9 +++-- tests/test_masks.py | 4 ++- tests/test_mnist_csv_format.py | 14 +++++--- tests/test_mnist_format.py | 9 +++-- tests/test_mot_format.py | 13 ++++--- tests/test_mots_format.py | 13 ++++--- tests/test_ndr.py | 7 ++-- tests/test_open_images_format.py | 12 ++++--- tests/test_ops.py | 17 ++++++---- tests/test_project.py | 17 ++++++---- tests/test_sampler.py | 11 +++--- tests/test_splitter.py | 15 +++----- tests/test_tfrecord_format.py | 25 ++++++++------ tests/test_transforms.py | 16 +++++---- tests/test_util.py | 6 ++-- tests/test_validator.py | 23 +++++++------ tests/test_vgg_face2_format.py | 14 +++++--- tests/test_voc_format.py | 30 ++++++++-------- tests/test_widerface_format.py | 10 ++++-- tests/test_yolo_format.py | 12 ++++--- 127 files changed, 680 insertions(+), 498 deletions(-) diff --git a/datum.py b/datum.py index 12c150bd16..d19ffe646a 100755 --- a/datum.py +++ b/datum.py @@ -3,6 +3,5 @@ from datumaro.cli.__main__ import main - if __name__ == '__main__': sys.exit(main()) diff --git a/datumaro/__main__.py b/datumaro/__main__.py index be1cb09298..062f6f0cd4 100644 --- a/datumaro/__main__.py +++ b/datumaro/__main__.py @@ -7,6 +7,5 @@ from datumaro.cli.__main__ import main - if __name__ == '__main__': sys.exit(main()) diff --git a/datumaro/cli/__main__.py b/datumaro/cli/__main__.py index 528c6d019e..b2113b6f1f 100644 --- a/datumaro/cli/__main__.py +++ b/datumaro/cli/__main__.py @@ -8,10 +8,9 @@ import os.path as osp import sys -from . import contexts, commands -from .util import CliException, add_subparser from ..version import VERSION - +from . import commands, contexts +from .util import CliException, add_subparser _log_levels = { 'debug': log.DEBUG, diff --git a/datumaro/cli/commands/__init__.py b/datumaro/cli/commands/__init__.py index 9324f12252..febb60775a 100644 --- a/datumaro/cli/commands/__init__.py +++ b/datumaro/cli/commands/__init__.py @@ -5,9 +5,6 @@ # pylint: disable=redefined-builtin from . import ( - create, add, remove, import_, - explain, - export, merge, convert, transform, filter, - diff, ediff, stats, - info, validate + add, convert, create, diff, ediff, explain, export, filter, import_, info, + merge, remove, stats, transform, validate, ) diff --git a/datumaro/cli/commands/convert.py b/datumaro/cli/commands/convert.py index 6d657d9b0b..bef04ef912 100644 --- a/datumaro/cli/commands/convert.py +++ b/datumaro/cli/commands/convert.py @@ -7,8 +7,8 @@ import os import os.path as osp -from datumaro.components.project import Environment from datumaro.components.dataset import Dataset +from datumaro.components.project import Environment from ..contexts.project import FilterModes from ..util import CliException, MultilineFormatter, make_file_name diff --git a/datumaro/cli/commands/explain.py b/datumaro/cli/commands/explain.py index dc4256194c..fe5c1dd39f 100644 --- a/datumaro/cli/commands/explain.py +++ b/datumaro/cli/commands/explain.py @@ -8,9 +8,12 @@ import os.path as osp from datumaro.components.project import Project -from datumaro.util.command_targets import (TargetKinds, target_selector, - ProjectTarget, SourceTarget, ImageTarget, is_project_path) +from datumaro.util.command_targets import ( + ImageTarget, ProjectTarget, SourceTarget, TargetKinds, is_project_path, + target_selector, +) from datumaro.util.image import load_image, save_image + from ..util import MultilineFormatter from ..util.project import load_project @@ -108,8 +111,8 @@ def explain_command(args): args.project_dir = osp.dirname(osp.abspath(args.target[1])) - import cv2 from matplotlib import cm + import cv2 project = load_project(args.project_dir) diff --git a/datumaro/cli/commands/merge.py b/datumaro/cli/commands/merge.py index 8a265035c3..fee44ff1a5 100644 --- a/datumaro/cli/commands/merge.py +++ b/datumaro/cli/commands/merge.py @@ -2,17 +2,17 @@ # # SPDX-License-Identifier: MIT +from collections import OrderedDict import argparse import json import logging as log import os.path as osp -from collections import OrderedDict -from datumaro.components.project import Project +from datumaro.components.errors import MergeError, QualityError from datumaro.components.operations import IntersectMerge -from datumaro.components.errors import QualityError, MergeError +from datumaro.components.project import Project -from ..util import at_least, MultilineFormatter, CliException +from ..util import CliException, MultilineFormatter, at_least from ..util.project import generate_next_file_name, load_project diff --git a/datumaro/cli/contexts/__init__.py b/datumaro/cli/contexts/__init__.py index b903435527..1598ff704c 100644 --- a/datumaro/cli/contexts/__init__.py +++ b/datumaro/cli/contexts/__init__.py @@ -3,4 +3,4 @@ # # SPDX-License-Identifier: MIT -from . import project, source, model \ No newline at end of file +from . import model, project, source diff --git a/datumaro/cli/contexts/model.py b/datumaro/cli/contexts/model.py index dfb2dc5ba9..20f1bffee3 100644 --- a/datumaro/cli/contexts/model.py +++ b/datumaro/cli/contexts/model.py @@ -12,8 +12,9 @@ from datumaro.util import error_rollback from ..util import CliException, MultilineFormatter, add_subparser -from ..util.project import load_project, \ - generate_next_name, generate_next_file_name +from ..util.project import ( + generate_next_file_name, generate_next_name, load_project, +) def build_add_parser(parser_ctor=argparse.ArgumentParser): diff --git a/datumaro/cli/contexts/project/__init__.py b/datumaro/cli/contexts/project/__init__.py index 64fd31e1aa..0e03593c2d 100644 --- a/datumaro/cli/contexts/project/__init__.py +++ b/datumaro/cli/contexts/project/__init__.py @@ -2,27 +2,30 @@ # # SPDX-License-Identifier: MIT +from enum import Enum import argparse import json import logging as log import os import os.path as osp import shutil -from enum import Enum + import numpy as np from datumaro.components.dataset_filter import DatasetItemEncoder from datumaro.components.extractor import AnnotationType -from datumaro.components.operations import (DistanceComparator, - ExactComparator, compute_ann_statistics, compute_image_statistics) -from datumaro.components.project import \ - PROJECT_DEFAULT_CONFIG as DEFAULT_CONFIG +from datumaro.components.operations import ( + DistanceComparator, ExactComparator, compute_ann_statistics, + compute_image_statistics, +) +from datumaro.components.project import PROJECT_DEFAULT_CONFIG as DEFAULT_CONFIG from datumaro.components.project import Environment, Project from datumaro.components.validator import TaskType from datumaro.util import error_rollback -from ...util import (CliException, MultilineFormatter, add_subparser, - make_file_name) +from ...util import ( + CliException, MultilineFormatter, add_subparser, make_file_name, +) from ...util.project import generate_next_file_name, load_project from .diff import DatasetDiffVisualizer diff --git a/datumaro/cli/contexts/project/diff.py b/datumaro/cli/contexts/project/diff.py index 1df1b8ab65..cb919e156f 100644 --- a/datumaro/cli/contexts/project/diff.py +++ b/datumaro/cli/contexts/project/diff.py @@ -4,12 +4,13 @@ # SPDX-License-Identifier: MIT from collections import Counter -from itertools import zip_longest from enum import Enum, auto +from itertools import zip_longest import logging as log import os import os.path as osp import warnings + import cv2 import numpy as np diff --git a/datumaro/cli/contexts/source.py b/datumaro/cli/contexts/source.py index caea28446c..54863340fe 100644 --- a/datumaro/cli/contexts/source.py +++ b/datumaro/cli/contexts/source.py @@ -9,7 +9,8 @@ import shutil from datumaro.components.project import Environment -from ..util import add_subparser, CliException, MultilineFormatter + +from ..util import CliException, MultilineFormatter, add_subparser from ..util.project import load_project diff --git a/datumaro/cli/util/__init__.py b/datumaro/cli/util/__init__.py index 0a4357f700..7fdded9cf9 100644 --- a/datumaro/cli/util/__init__.py +++ b/datumaro/cli/util/__init__.py @@ -68,7 +68,8 @@ def make_file_name(s): Normalizes string, converts to lowercase, removes non-alpha characters, and converts spaces to hyphens. """ - import unicodedata, re + import re + import unicodedata s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore') s = s.decode() s = re.sub(r'[^\w\s-]', '', s).strip().lower() diff --git a/datumaro/components/algorithms/rise.py b/datumaro/components/algorithms/rise.py index 0a4c39d8a2..f2ffd4604d 100644 --- a/datumaro/components/algorithms/rise.py +++ b/datumaro/components/algorithms/rise.py @@ -6,6 +6,7 @@ # pylint: disable=unused-variable from math import ceil + import numpy as np from datumaro.components.extractor import AnnotationType diff --git a/datumaro/components/cli_plugin.py b/datumaro/components/cli_plugin.py index ca23de1b4d..76b54be197 100644 --- a/datumaro/components/cli_plugin.py +++ b/datumaro/components/cli_plugin.py @@ -20,10 +20,11 @@ def _get_name(cls): def _get_doc(cls): doc = getattr(cls, '__doc__', "") if doc: - from datumaro.components.launcher import Launcher - from datumaro.components.extractor import Extractor, Transform, \ - Importer from datumaro.components.converter import Converter + from datumaro.components.extractor import ( + Extractor, Importer, Transform, + ) + from datumaro.components.launcher import Launcher base_classes = [Launcher, Extractor, Transform, Importer, Converter] if any(getattr(t, '__doc__', '') == doc for t in base_classes): diff --git a/datumaro/components/config_model.py b/datumaro/components/config_model.py index 49f85e9133..8cc536b969 100644 --- a/datumaro/components/config_model.py +++ b/datumaro/components/config_model.py @@ -3,10 +3,9 @@ # # SPDX-License-Identifier: MIT -from datumaro.components.config import Config, \ - DictConfig as _DictConfig, \ - SchemaBuilder as _SchemaBuilder - +from datumaro.components.config import Config +from datumaro.components.config import DictConfig as _DictConfig +from datumaro.components.config import SchemaBuilder as _SchemaBuilder SOURCE_SCHEMA = _SchemaBuilder() \ .add('url', str) \ diff --git a/datumaro/components/converter.py b/datumaro/components/converter.py index a63f7bd1f7..eafd23da62 100644 --- a/datumaro/components/converter.py +++ b/datumaro/components/converter.py @@ -9,7 +9,7 @@ import shutil from datumaro.components.cli_plugin import CliPlugin -from datumaro.util.image import save_image, ByteImage +from datumaro.util.image import ByteImage, save_image class Converter(CliPlugin): diff --git a/datumaro/components/dataset.py b/datumaro/components/dataset.py index 2f05b2f75d..3be34a4a43 100644 --- a/datumaro/components/dataset.py +++ b/datumaro/components/dataset.py @@ -2,28 +2,30 @@ # # SPDX-License-Identifier: MIT -from copy import copy from contextlib import contextmanager +from copy import copy from enum import Enum, auto -from typing import Any, Iterable, Iterator, Optional, Tuple, Union, Dict, List -import logging as log +from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, Union import inspect +import logging as log import os import os.path as osp import shutil -from datumaro.components.dataset_filter import \ - XPathDatasetFilter, XPathAnnotationsFilter -from datumaro.components.extractor import (CategoriesInfo, Extractor, - IExtractor, ItemTransform, LabelCategories, AnnotationType, DatasetItem, - DEFAULT_SUBSET_NAME, Transform) +from datumaro.components.dataset_filter import ( + XPathAnnotationsFilter, XPathDatasetFilter, +) from datumaro.components.environment import Environment -from datumaro.components.errors import (CategoriesRedefinedError, - DatumaroError, RepeatedItemError) +from datumaro.components.errors import ( + CategoriesRedefinedError, DatumaroError, RepeatedItemError, +) +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, AnnotationType, CategoriesInfo, DatasetItem, Extractor, + IExtractor, ItemTransform, LabelCategories, Transform, +) from datumaro.util import error_rollback, is_member_redefined from datumaro.util.log_utils import logging_disabled - DEFAULT_FORMAT = 'datumaro' IDataset = IExtractor diff --git a/datumaro/components/dataset_filter.py b/datumaro/components/dataset_filter.py index fbaf60c5ce..dd06919edd 100644 --- a/datumaro/components/dataset_filter.py +++ b/datumaro/components/dataset_filter.py @@ -4,10 +4,12 @@ # SPDX-License-Identifier: MIT import logging as log -from lxml import etree as ET # lxml has proper XPath implementation -from datumaro.components.extractor import (ItemTransform, - Annotation, AnnotationType, - Label, Mask, Points, Polygon, PolyLine, Bbox, Caption, + +from lxml import etree as ET # lxml has proper XPath implementation + +from datumaro.components.extractor import ( + Annotation, AnnotationType, Bbox, Caption, ItemTransform, Label, Mask, + Points, Polygon, PolyLine, ) diff --git a/datumaro/components/environment.py b/datumaro/components/environment.py index 4d1bca7042..0170bf979d 100644 --- a/datumaro/components/environment.py +++ b/datumaro/components/environment.py @@ -4,11 +4,12 @@ from functools import partial from glob import glob +from typing import Dict, Iterable import inspect import logging as log import os import os.path as osp -from typing import Dict, Iterable + import git from datumaro.components.config import Config @@ -112,7 +113,8 @@ class Environment: def __init__(self, config=None): from datumaro.components.project import ( - PROJECT_DEFAULT_CONFIG, PROJECT_SCHEMA) + PROJECT_DEFAULT_CONFIG, PROJECT_SCHEMA, + ) config = Config(config, fallback=PROJECT_DEFAULT_CONFIG, schema=PROJECT_SCHEMA) @@ -128,8 +130,9 @@ def _filter(accept, skip=None): return lambda t: issubclass(t, accept) and t not in skip from datumaro.components.converter import Converter - from datumaro.components.extractor import (Importer, Extractor, - SourceExtractor, Transform, ItemTransform) + from datumaro.components.extractor import ( + Extractor, Importer, ItemTransform, SourceExtractor, Transform, + ) from datumaro.components.launcher import Launcher self._extractors = PluginRegistry(_filter(Extractor, skip=SourceExtractor)) @@ -207,8 +210,9 @@ def _import_module(cls, module_dir, module_name, types, package=None): def _load_plugins(cls, plugins_dir, types=None): if not types: from datumaro.components.converter import Converter - from datumaro.components.extractor import (Extractor, Importer, - Transform) + from datumaro.components.extractor import ( + Extractor, Importer, Transform, + ) from datumaro.components.launcher import Launcher types = [Extractor, Converter, Importer, Launcher, Transform] diff --git a/datumaro/components/extractor.py b/datumaro/components/extractor.py index ea5a447ac6..0b6c81b6a0 100644 --- a/datumaro/components/extractor.py +++ b/datumaro/components/extractor.py @@ -5,16 +5,16 @@ from enum import Enum, auto from glob import iglob -from typing import Callable, Iterable, List, Dict, Optional +from typing import Callable, Dict, Iterable, List, Optional import os import os.path as osp -from attr import attrs, attrib +from attr import attrib, attrs import attr import numpy as np +from datumaro.util.attrs_util import default_if_none, not_empty from datumaro.util.image import Image -from datumaro.util.attrs_util import not_empty, default_if_none class AnnotationType(Enum): @@ -655,7 +655,7 @@ def find_sources(cls, path) -> List[Dict]: raise NotImplementedError() def __call__(self, path, **extra_params): - from datumaro.components.project import Project # cyclic import + from datumaro.components.project import Project # cyclic import project = Project() sources = self.find_sources(osp.normpath(path)) diff --git a/datumaro/components/launcher.py b/datumaro/components/launcher.py index 8f05b36bee..9e376b14ba 100644 --- a/datumaro/components/launcher.py +++ b/datumaro/components/launcher.py @@ -5,8 +5,9 @@ import numpy as np -from datumaro.components.extractor import (Transform, LabelCategories, - AnnotationType) +from datumaro.components.extractor import ( + AnnotationType, LabelCategories, Transform, +) from datumaro.util import take_by diff --git a/datumaro/components/operations.py b/datumaro/components/operations.py index 00ba14394c..bb95d884c2 100644 --- a/datumaro/components/operations.py +++ b/datumaro/components/operations.py @@ -4,27 +4,32 @@ from collections import OrderedDict from copy import deepcopy +from unittest import TestCase import hashlib import logging as log -from unittest import TestCase +from attr import attrib, attrs import attr import cv2 import numpy as np -from attr import attrib, attrs from datumaro.components.cli_plugin import CliPlugin -from datumaro.util import find, filter_dict -from datumaro.components.extractor import (AnnotationType, Bbox, - CategoriesInfo, Label, - LabelCategories, PointsCategories, MaskCategories) -from datumaro.components.errors import (DatumaroError, FailedAttrVotingError, - FailedLabelVotingError, MismatchingImageInfoError, NoMatchingAnnError, - NoMatchingItemError, AnnotationsTooCloseError, WrongGroupError) from datumaro.components.dataset import Dataset, DatasetItemStorage -from datumaro.util.attrs_util import ensure_cls, default_if_none -from datumaro.util.annotation_util import (segment_iou, bbox_iou, - mean_bbox, OKS, find_instances, max_bbox, smooth_line) +from datumaro.components.errors import ( + AnnotationsTooCloseError, DatumaroError, FailedAttrVotingError, + FailedLabelVotingError, MismatchingImageInfoError, NoMatchingAnnError, + NoMatchingItemError, WrongGroupError, +) +from datumaro.components.extractor import ( + AnnotationType, Bbox, CategoriesInfo, Label, LabelCategories, + MaskCategories, PointsCategories, +) +from datumaro.util import filter_dict, find +from datumaro.util.annotation_util import ( + OKS, bbox_iou, find_instances, max_bbox, mean_bbox, segment_iou, + smooth_line, +) +from datumaro.util.attrs_util import default_if_none, ensure_cls def get_ann_type(anns, t): diff --git a/datumaro/components/project.py b/datumaro/components/project.py index 4d0cc4a793..8996ffce92 100644 --- a/datumaro/components/project.py +++ b/datumaro/components/project.py @@ -9,11 +9,13 @@ import shutil from datumaro.components.config import Config -from datumaro.components.config_model import (Model, Source, - PROJECT_DEFAULT_CONFIG, PROJECT_SCHEMA) -from datumaro.components.dataset import (IDataset, Dataset, DEFAULT_FORMAT) -from datumaro.components.dataset_filter import (XPathAnnotationsFilter, - XPathDatasetFilter) +from datumaro.components.config_model import ( + PROJECT_DEFAULT_CONFIG, PROJECT_SCHEMA, Model, Source, +) +from datumaro.components.dataset import DEFAULT_FORMAT, Dataset, IDataset +from datumaro.components.dataset_filter import ( + XPathAnnotationsFilter, XPathDatasetFilter, +) from datumaro.components.environment import Environment from datumaro.components.errors import DatumaroError from datumaro.components.extractor import DEFAULT_SUBSET_NAME, Extractor diff --git a/datumaro/plugins/accuracy_checker_plugin/details/ac.py b/datumaro/plugins/accuracy_checker_plugin/details/ac.py index b235e57848..63ce0c5fdb 100644 --- a/datumaro/plugins/accuracy_checker_plugin/details/ac.py +++ b/datumaro/plugins/accuracy_checker_plugin/details/ac.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: MIT from datumaro.util.tf_util import import_tf + import_tf() # prevent TF loading and potential interpreter crash from itertools import groupby diff --git a/datumaro/plugins/accuracy_checker_plugin/details/representation.py b/datumaro/plugins/accuracy_checker_plugin/details/representation.py index d502dd2b3a..94659e7aec 100644 --- a/datumaro/plugins/accuracy_checker_plugin/details/representation.py +++ b/datumaro/plugins/accuracy_checker_plugin/details/representation.py @@ -4,12 +4,14 @@ # SPDX-License-Identifier: MIT from datumaro.util.tf_util import import_tf + import_tf() # prevent TF loading and potential interpreter crash import accuracy_checker.representation as ac -import datumaro.components.extractor as dm from datumaro.util.annotation_util import softmax +import datumaro.components.extractor as dm + def import_predictions(predictions): # Convert Accuracy checker predictions to Datumaro annotations diff --git a/datumaro/plugins/accuracy_checker_plugin/launcher.py b/datumaro/plugins/accuracy_checker_plugin/launcher.py index 4d94a3b4b5..3ae367aeb8 100644 --- a/datumaro/plugins/accuracy_checker_plugin/launcher.py +++ b/datumaro/plugins/accuracy_checker_plugin/launcher.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: MIT import os.path as osp + import yaml from datumaro.components.cli_plugin import CliPlugin diff --git a/datumaro/plugins/camvid_format.py b/datumaro/plugins/camvid_format.py index ee05d7b1aa..cbb483f9bd 100644 --- a/datumaro/plugins/camvid_format.py +++ b/datumaro/plugins/camvid_format.py @@ -3,18 +3,19 @@ # # SPDX-License-Identifier: MIT +from collections import OrderedDict +from enum import Enum, auto import logging as log import os import os.path as osp -from collections import OrderedDict -from enum import Enum, auto import numpy as np from datumaro.components.converter import Converter -from datumaro.components.extractor import (AnnotationType, CompiledMask, - DatasetItem, Importer, LabelCategories, Mask, - MaskCategories, SourceExtractor) +from datumaro.components.extractor import ( + AnnotationType, CompiledMask, DatasetItem, Importer, LabelCategories, Mask, + MaskCategories, SourceExtractor, +) from datumaro.util import find, str_to_bool from datumaro.util.annotation_util import make_label_id_mapping from datumaro.util.image import save_image diff --git a/datumaro/plugins/cifar_format.py b/datumaro/plugins/cifar_format.py index 8212911710..b4dcdc616e 100644 --- a/datumaro/plugins/cifar_format.py +++ b/datumaro/plugins/cifar_format.py @@ -4,12 +4,15 @@ import os import os.path as osp -import pickle # nosec - disable B403:import_pickle check +import pickle # nosec - disable B403:import_pickle check import numpy as np + from datumaro.components.converter import Converter -from datumaro.components.extractor import (AnnotationType, DatasetItem, - Importer, Label, LabelCategories, SourceExtractor) +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Importer, Label, LabelCategories, + SourceExtractor, +) from datumaro.util import cast diff --git a/datumaro/plugins/cityscapes_format.py b/datumaro/plugins/cityscapes_format.py index 64d5eb2a41..dbee3fd171 100644 --- a/datumaro/plugins/cityscapes_format.py +++ b/datumaro/plugins/cityscapes_format.py @@ -3,25 +3,25 @@ # # SPDX-License-Identifier: MIT -import logging as log -import os -import os.path as osp from collections import OrderedDict from enum import Enum, auto from glob import iglob +import logging as log +import os +import os.path as osp import numpy as np from datumaro.components.converter import Converter -from datumaro.components.extractor import (AnnotationType, CompiledMask, - DatasetItem, Importer, LabelCategories, Mask, - MaskCategories, SourceExtractor) +from datumaro.components.extractor import ( + AnnotationType, CompiledMask, DatasetItem, Importer, LabelCategories, Mask, + MaskCategories, SourceExtractor, +) from datumaro.util import str_to_bool from datumaro.util.annotation_util import make_label_id_mapping -from datumaro.util.image import save_image, load_image +from datumaro.util.image import load_image, save_image from datumaro.util.mask_tools import generate_colormap, paint_mask - CityscapesLabelMap = OrderedDict([ ('unlabeled', (0, 0, 0)), ('egovehicle', (0, 0, 0)), diff --git a/datumaro/plugins/coco_format/converter.py b/datumaro/plugins/coco_format/converter.py index aadb8e1519..4d444a2165 100644 --- a/datumaro/plugins/coco_format/converter.py +++ b/datumaro/plugins/coco_format/converter.py @@ -3,27 +3,29 @@ # # SPDX-License-Identifier: MIT +from enum import Enum, auto +from itertools import chain, groupby import json import logging as log import os import os.path as osp -from enum import Enum, auto -from itertools import chain, groupby -import numpy as np +import numpy as np import pycocotools.mask as mask_utils -import datumaro.util.annotation_util as anno_tools -import datumaro.util.mask_tools as mask_tools from datumaro.components.converter import Converter -from datumaro.components.extractor import (DatasetItem, - _COORDINATE_ROUNDING_DIGITS, AnnotationType, Points) from datumaro.components.dataset import ItemStatus +from datumaro.components.extractor import ( + _COORDINATE_ROUNDING_DIGITS, AnnotationType, DatasetItem, Points, +) from datumaro.util import cast, find, str_to_bool from datumaro.util.image import save_image +import datumaro.util.annotation_util as anno_tools +import datumaro.util.mask_tools as mask_tools from .format import CocoPath, CocoTask + class SegmentationMode(Enum): guess = auto() polygons = auto() diff --git a/datumaro/plugins/coco_format/extractor.py b/datumaro/plugins/coco_format/extractor.py index a2210dfa8a..303f0e1333 100644 --- a/datumaro/plugins/coco_format/extractor.py +++ b/datumaro/plugins/coco_format/extractor.py @@ -11,15 +11,15 @@ from pycocotools.coco import COCO import pycocotools.mask as mask_utils -from datumaro.components.extractor import (CompiledMask, Mask, SourceExtractor, - DEFAULT_SUBSET_NAME, DatasetItem, - AnnotationType, Label, RleMask, Points, Polygon, Bbox, Caption, - LabelCategories, PointsCategories +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, AnnotationType, Bbox, Caption, CompiledMask, + DatasetItem, Label, LabelCategories, Mask, Points, PointsCategories, + Polygon, RleMask, SourceExtractor, ) from datumaro.util.image import Image, lazy_image, load_image from datumaro.util.mask_tools import bgr2index -from .format import CocoTask, CocoPath +from .format import CocoPath, CocoTask class _CocoExtractor(SourceExtractor): diff --git a/datumaro/plugins/coco_format/importer.py b/datumaro/plugins/coco_format/importer.py index f2c68a1f74..ca76dc55db 100644 --- a/datumaro/plugins/coco_format/importer.py +++ b/datumaro/plugins/coco_format/importer.py @@ -31,7 +31,7 @@ def detect(cls, path): return len(cls.find_sources(path)) != 0 def __call__(self, path, **extra_params): - from datumaro.components.project import Project # cyclic import + from datumaro.components.project import Project # cyclic import project = Project() subsets = self.find_sources(path) diff --git a/datumaro/plugins/cvat_format/converter.py b/datumaro/plugins/cvat_format/converter.py index 08bf85f58b..0257ea3bf1 100644 --- a/datumaro/plugins/cvat_format/converter.py +++ b/datumaro/plugins/cvat_format/converter.py @@ -3,17 +3,18 @@ # # SPDX-License-Identifier: MIT -import logging as log -import os -import os.path as osp from collections import OrderedDict from itertools import chain from xml.sax.saxutils import XMLGenerator +import logging as log +import os +import os.path as osp from datumaro.components.converter import Converter from datumaro.components.dataset import ItemStatus -from datumaro.components.extractor import (AnnotationType, DatasetItem, - LabelCategories) +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, LabelCategories, +) from datumaro.util import cast, pairs from .format import CvatPath diff --git a/datumaro/plugins/cvat_format/extractor.py b/datumaro/plugins/cvat_format/extractor.py index 466ab96a95..cf17cbf995 100644 --- a/datumaro/plugins/cvat_format/extractor.py +++ b/datumaro/plugins/cvat_format/extractor.py @@ -5,11 +5,12 @@ from collections import OrderedDict import os.path as osp + from defusedxml import ElementTree -from datumaro.components.extractor import (SourceExtractor, DatasetItem, - AnnotationType, Points, Polygon, PolyLine, Bbox, Label, - LabelCategories, Importer +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Importer, Label, LabelCategories, Points, + Polygon, PolyLine, SourceExtractor, ) from datumaro.util.image import Image diff --git a/datumaro/plugins/datumaro_format/converter.py b/datumaro/plugins/datumaro_format/converter.py index 45cc4c75f6..787fe9f555 100644 --- a/datumaro/plugins/datumaro_format/converter.py +++ b/datumaro/plugins/datumaro_format/converter.py @@ -8,15 +8,16 @@ import json import os import os.path as osp -import numpy as np +import numpy as np import pycocotools.mask as mask_utils + from datumaro.components.converter import Converter from datumaro.components.dataset import ItemStatus from datumaro.components.extractor import ( - DEFAULT_SUBSET_NAME, Annotation, DatasetItem, _Shape, - Label, Mask, RleMask, Points, Polygon, PolyLine, Bbox, Caption, - LabelCategories, MaskCategories, PointsCategories + DEFAULT_SUBSET_NAME, Annotation, Bbox, Caption, DatasetItem, Label, + LabelCategories, Mask, MaskCategories, Points, PointsCategories, Polygon, + PolyLine, RleMask, _Shape, ) from datumaro.util import cast diff --git a/datumaro/plugins/datumaro_format/extractor.py b/datumaro/plugins/datumaro_format/extractor.py index 8b4799c927..cf1c642b5e 100644 --- a/datumaro/plugins/datumaro_format/extractor.py +++ b/datumaro/plugins/datumaro_format/extractor.py @@ -6,9 +6,10 @@ import json import os.path as osp -from datumaro.components.extractor import (SourceExtractor, DatasetItem, - AnnotationType, Label, RleMask, Points, Polygon, PolyLine, Bbox, Caption, - LabelCategories, MaskCategories, PointsCategories, Importer +from datumaro.components.extractor import ( + AnnotationType, Bbox, Caption, DatasetItem, Importer, Label, + LabelCategories, MaskCategories, Points, PointsCategories, Polygon, + PolyLine, RleMask, SourceExtractor, ) from datumaro.util.image import Image diff --git a/datumaro/plugins/icdar_format/extractor.py b/datumaro/plugins/icdar_format/extractor.py index bc6f4e91c4..7bc85d9f7f 100644 --- a/datumaro/plugins/icdar_format/extractor.py +++ b/datumaro/plugins/icdar_format/extractor.py @@ -7,8 +7,10 @@ import numpy as np -from datumaro.components.extractor import (Bbox, Caption, DatasetItem, - Importer, Mask, MaskCategories, Polygon, SourceExtractor) +from datumaro.components.extractor import ( + Bbox, Caption, DatasetItem, Importer, Mask, MaskCategories, Polygon, + SourceExtractor, +) from datumaro.util.image import find_images from datumaro.util.mask_tools import lazy_mask diff --git a/datumaro/plugins/image_dir_format.py b/datumaro/plugins/image_dir_format.py index 3cca401a43..c8d52f03e5 100644 --- a/datumaro/plugins/image_dir_format.py +++ b/datumaro/plugins/image_dir_format.py @@ -7,8 +7,8 @@ import os import os.path as osp -from datumaro.components.extractor import DatasetItem, SourceExtractor, Importer from datumaro.components.converter import Converter +from datumaro.components.extractor import DatasetItem, Importer, SourceExtractor from datumaro.util.image import find_images diff --git a/datumaro/plugins/image_zip_format.py b/datumaro/plugins/image_zip_format.py index 834a766a6d..5b0bd59898 100644 --- a/datumaro/plugins/image_zip_format.py +++ b/datumaro/plugins/image_zip_format.py @@ -3,17 +3,17 @@ # SPDX-License-Identifier: MIT from enum import Enum +from zipfile import ZIP_BZIP2, ZIP_DEFLATED, ZIP_LZMA, ZIP_STORED, ZipFile import logging as log import os import os.path as osp -from zipfile import ZIP_BZIP2, ZIP_DEFLATED, ZIP_LZMA, ZIP_STORED, ZipFile from datumaro.components.converter import Converter from datumaro.components.extractor import DatasetItem, Importer, SourceExtractor - from datumaro.util import parse_str_enum_value from datumaro.util.image import IMAGE_EXTENSIONS, ByteImage, encode_image + class Compression(Enum): ZIP_STORED = ZIP_STORED ZIP_DEFLATED = ZIP_DEFLATED diff --git a/datumaro/plugins/imagenet_format.py b/datumaro/plugins/imagenet_format.py index 9254662d06..b9223a33fa 100644 --- a/datumaro/plugins/imagenet_format.py +++ b/datumaro/plugins/imagenet_format.py @@ -6,10 +6,11 @@ import os import os.path as osp -from datumaro.components.extractor import (DatasetItem, Label, - LabelCategories, AnnotationType, SourceExtractor, Importer -) from datumaro.components.converter import Converter +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Importer, Label, LabelCategories, + SourceExtractor, +) from datumaro.util.image import find_images diff --git a/datumaro/plugins/imagenet_txt_format.py b/datumaro/plugins/imagenet_txt_format.py index d50148956f..a66868d408 100644 --- a/datumaro/plugins/imagenet_txt_format.py +++ b/datumaro/plugins/imagenet_txt_format.py @@ -6,10 +6,11 @@ import os import os.path as osp -from datumaro.components.extractor import (DatasetItem, Label, - LabelCategories, AnnotationType, SourceExtractor, Importer -) from datumaro.components.converter import Converter +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Importer, Label, LabelCategories, + SourceExtractor, +) class ImagenetTxtPath: diff --git a/datumaro/plugins/kitti_format/converter.py b/datumaro/plugins/kitti_format/converter.py index 661d642891..a11c485211 100644 --- a/datumaro/plugins/kitti_format/converter.py +++ b/datumaro/plugins/kitti_format/converter.py @@ -3,27 +3,29 @@ # # SPDX-License-Identifier: MIT +from collections import OrderedDict +from enum import Enum, auto import logging as log import os import os.path as osp -from collections import OrderedDict -from enum import Enum, auto import numpy as np from datumaro.components.converter import Converter -from datumaro.components.extractor import (AnnotationType, - CompiledMask, LabelCategories) -from datumaro.util import parse_str_enum_value, str_to_bool, cast +from datumaro.components.extractor import ( + AnnotationType, CompiledMask, LabelCategories, +) +from datumaro.util import cast, parse_str_enum_value, str_to_bool from datumaro.util.annotation_util import make_label_id_mapping from datumaro.util.image import save_image from datumaro.util.mask_tools import paint_mask -from .format import (KittiTask, KittiPath, KittiLabelMap, - make_kitti_categories, make_kitti_detection_categories, - parse_label_map, write_label_map, +from .format import ( + KittiLabelMap, KittiPath, KittiTask, make_kitti_categories, + make_kitti_detection_categories, parse_label_map, write_label_map, ) + class LabelmapType(Enum): kitti = auto() source = auto() diff --git a/datumaro/plugins/kitti_format/extractor.py b/datumaro/plugins/kitti_format/extractor.py index f9ad6fd6fc..7478587671 100644 --- a/datumaro/plugins/kitti_format/extractor.py +++ b/datumaro/plugins/kitti_format/extractor.py @@ -6,15 +6,17 @@ import numpy as np -from datumaro.components.extractor import (SourceExtractor, - AnnotationType, DatasetItem, Mask, Bbox) -from datumaro.util.image import load_image, find_images +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Mask, SourceExtractor, +) +from datumaro.util.image import find_images, load_image from .format import ( - KittiTask, KittiPath, KittiLabelMap, parse_label_map, - make_kitti_categories, make_kitti_detection_categories + KittiLabelMap, KittiPath, KittiTask, make_kitti_categories, + make_kitti_detection_categories, parse_label_map, ) + class _KittiExtractor(SourceExtractor): def __init__(self, path, task, subset=None): assert osp.isdir(path), path diff --git a/datumaro/plugins/kitti_format/format.py b/datumaro/plugins/kitti_format/format.py index 512eaa453a..5bcfdb79d8 100644 --- a/datumaro/plugins/kitti_format/format.py +++ b/datumaro/plugins/kitti_format/format.py @@ -6,10 +6,9 @@ from collections import OrderedDict from enum import Enum, auto -from datumaro.components.extractor import (AnnotationType, - LabelCategories, MaskCategories +from datumaro.components.extractor import ( + AnnotationType, LabelCategories, MaskCategories, ) - from datumaro.util.mask_tools import generate_colormap diff --git a/datumaro/plugins/kitti_format/importer.py b/datumaro/plugins/kitti_format/importer.py index b9e40d73cf..669896c98c 100644 --- a/datumaro/plugins/kitti_format/importer.py +++ b/datumaro/plugins/kitti_format/importer.py @@ -10,7 +10,8 @@ from datumaro.components.extractor import Importer from datumaro.util.log_utils import logging_disabled -from .format import KittiTask, KittiPath +from .format import KittiPath, KittiTask + class KittiImporter(Importer): _TASKS = { @@ -24,7 +25,7 @@ def detect(cls, path): return len(cls.find_sources(path)) != 0 def __call__(self, path, **extra_params): - from datumaro.components.project import Project # cyclic import + from datumaro.components.project import Project # cyclic import project = Project() subsets = self.find_sources(path) diff --git a/datumaro/plugins/labelme_format.py b/datumaro/plugins/labelme_format.py index 8c6fc3c3e8..f95daf4aea 100644 --- a/datumaro/plugins/labelme_format.py +++ b/datumaro/plugins/labelme_format.py @@ -8,16 +8,19 @@ import logging as log import os import os.path as osp + from defusedxml import ElementTree import numpy as np -from datumaro.components.extractor import (Extractor, Importer, - DatasetItem, AnnotationType, Mask, Bbox, Polygon, LabelCategories) from datumaro.components.converter import Converter +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Extractor, Importer, LabelCategories, + Mask, Polygon, +) from datumaro.util import cast, escape, unescape -from datumaro.util.os_util import split_path from datumaro.util.image import Image, save_image -from datumaro.util.mask_tools import load_mask, find_mask_bbox +from datumaro.util.mask_tools import find_mask_bbox, load_mask +from datumaro.util.os_util import split_path class LabelMePath: diff --git a/datumaro/plugins/lfw_format.py b/datumaro/plugins/lfw_format.py index 004c4e9315..59aef70ec3 100644 --- a/datumaro/plugins/lfw_format.py +++ b/datumaro/plugins/lfw_format.py @@ -7,8 +7,10 @@ import re from datumaro.components.converter import Converter -from datumaro.components.extractor import (AnnotationType, DatasetItem, - Importer, Label, LabelCategories, Points, SourceExtractor) +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Importer, Label, LabelCategories, Points, + SourceExtractor, +) from datumaro.util.image import find_images diff --git a/datumaro/plugins/market1501_format.py b/datumaro/plugins/market1501_format.py index 2493b495bc..5f2cbfe138 100644 --- a/datumaro/plugins/market1501_format.py +++ b/datumaro/plugins/market1501_format.py @@ -2,15 +2,14 @@ # # SPDX-License-Identifier: MIT +from distutils.util import strtobool +from itertools import chain import os import os.path as osp import re -from distutils.util import strtobool -from itertools import chain from datumaro.components.converter import Converter -from datumaro.components.extractor import (DatasetItem, Importer, - SourceExtractor) +from datumaro.components.extractor import DatasetItem, Importer, SourceExtractor from datumaro.util.image import find_images diff --git a/datumaro/plugins/mnist_csv_format.py b/datumaro/plugins/mnist_csv_format.py index ae0fa8bf8c..17e257d263 100644 --- a/datumaro/plugins/mnist_csv_format.py +++ b/datumaro/plugins/mnist_csv_format.py @@ -6,9 +6,12 @@ import os.path as osp import numpy as np + from datumaro.components.converter import Converter -from datumaro.components.extractor import (AnnotationType, DatasetItem, - Importer, Label, LabelCategories, SourceExtractor) +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Importer, Label, LabelCategories, + SourceExtractor, +) class MnistCsvPath: diff --git a/datumaro/plugins/mnist_format.py b/datumaro/plugins/mnist_format.py index 0cd97b06df..2eb4c474bc 100644 --- a/datumaro/plugins/mnist_format.py +++ b/datumaro/plugins/mnist_format.py @@ -7,9 +7,12 @@ import os.path as osp import numpy as np + from datumaro.components.converter import Converter -from datumaro.components.extractor import (AnnotationType, DatasetItem, - Importer, Label, LabelCategories, SourceExtractor) +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Importer, Label, LabelCategories, + SourceExtractor, +) class MnistPath: diff --git a/datumaro/plugins/mot_format.py b/datumaro/plugins/mot_format.py index 8008f25d2b..abd1e95fc7 100644 --- a/datumaro/plugins/mot_format.py +++ b/datumaro/plugins/mot_format.py @@ -7,20 +7,20 @@ # Another description: https://motchallenge.net/instructions from collections import OrderedDict -import csv from enum import Enum +import csv import logging as log import os import os.path as osp -from datumaro.components.extractor import (SourceExtractor, Importer, - DatasetItem, AnnotationType, Bbox, LabelCategories -) from datumaro.components.converter import Converter +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Importer, LabelCategories, + SourceExtractor, +) from datumaro.util import cast from datumaro.util.image import Image, find_images - MotLabel = Enum('MotLabel', [ ('pedestrian', 1), ('person on vehicle', 2), diff --git a/datumaro/plugins/mots_format.py b/datumaro/plugins/mots_format.py index 7728efd9e1..16a632ba06 100644 --- a/datumaro/plugins/mots_format.py +++ b/datumaro/plugins/mots_format.py @@ -9,12 +9,14 @@ import logging as log import os import os.path as osp + import numpy as np -from datumaro.components.extractor import (SourceExtractor, Importer, - DatasetItem, AnnotationType, Mask, LabelCategories -) from datumaro.components.converter import Converter +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Importer, LabelCategories, Mask, + SourceExtractor, +) from datumaro.util.image import find_images, load_image, save_image from datumaro.util.mask_tools import merge_masks diff --git a/datumaro/plugins/ndr.py b/datumaro/plugins/ndr.py index 3f2610d899..cd6f0112e5 100644 --- a/datumaro/plugins/ndr.py +++ b/datumaro/plugins/ndr.py @@ -5,12 +5,12 @@ from enum import Enum, auto import logging as log +from scipy.linalg import orth import cv2 import numpy as np -from scipy.linalg import orth -from datumaro.components.extractor import Transform, DEFAULT_SUBSET_NAME from datumaro.components.cli_plugin import CliPlugin +from datumaro.components.extractor import DEFAULT_SUBSET_NAME, Transform from datumaro.util import parse_str_enum_value diff --git a/datumaro/plugins/open_images_format.py b/datumaro/plugins/open_images_format.py index d6b83f33c5..466a5571d2 100644 --- a/datumaro/plugins/open_images_format.py +++ b/datumaro/plugins/open_images_format.py @@ -16,9 +16,11 @@ from attr import attrs from datumaro.components.converter import Converter -from datumaro.components.errors import DatasetError, RepeatedItemError, UndefinedLabel +from datumaro.components.errors import ( + DatasetError, RepeatedItemError, UndefinedLabel, +) from datumaro.components.extractor import ( - AnnotationType, DatasetItem, Importer, Label, LabelCategories, Extractor, + AnnotationType, DatasetItem, Extractor, Importer, Label, LabelCategories, ) from datumaro.components.validator import Severity from datumaro.util.image import find_images diff --git a/datumaro/plugins/openvino_plugin/launcher.py b/datumaro/plugins/openvino_plugin/launcher.py index 1617e0f142..a5767afbf4 100644 --- a/datumaro/plugins/openvino_plugin/launcher.py +++ b/datumaro/plugins/openvino_plugin/launcher.py @@ -5,13 +5,13 @@ # pylint: disable=exec-used +import logging as log import os.path as osp import shutil -import logging as log -import cv2 -import numpy as np from openvino.inference_engine import IECore +import cv2 +import numpy as np from datumaro.components.cli_plugin import CliPlugin from datumaro.components.launcher import Launcher diff --git a/datumaro/plugins/openvino_plugin/samples/mobilenet_v2_pytorch_interp.py b/datumaro/plugins/openvino_plugin/samples/mobilenet_v2_pytorch_interp.py index 41df21e466..a765dd42b1 100644 --- a/datumaro/plugins/openvino_plugin/samples/mobilenet_v2_pytorch_interp.py +++ b/datumaro/plugins/openvino_plugin/samples/mobilenet_v2_pytorch_interp.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: MIT -from datumaro.components.extractor import Label, LabelCategories, AnnotationType +from datumaro.components.extractor import AnnotationType, Label, LabelCategories from datumaro.util.annotation_util import softmax diff --git a/datumaro/plugins/sampler/sampler.py b/datumaro/plugins/sampler/sampler.py index 2fefa0e1dc..797fd800a5 100644 --- a/datumaro/plugins/sampler/sampler.py +++ b/datumaro/plugins/sampler/sampler.py @@ -6,10 +6,10 @@ import pandas as pd -from datumaro.components.extractor import Transform from datumaro.components.cli_plugin import CliPlugin +from datumaro.components.extractor import Transform -from .algorithm.algorithm import SamplingMethod, Algorithm +from .algorithm.algorithm import Algorithm, SamplingMethod class Sampler(Transform, CliPlugin): diff --git a/datumaro/plugins/splitter.py b/datumaro/plugins/splitter.py index f983715cd8..1a5a1b4d8a 100644 --- a/datumaro/plugins/splitter.py +++ b/datumaro/plugins/splitter.py @@ -2,15 +2,17 @@ # # SPDX-License-Identifier: MIT -import logging as log -import copy -from math import gcd from enum import Enum, auto +from math import gcd +import copy +import logging as log + import numpy as np -from datumaro.components.extractor import (Transform, AnnotationType, - DEFAULT_SUBSET_NAME) from datumaro.components.cli_plugin import CliPlugin +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, AnnotationType, Transform, +) from datumaro.util import cast NEAR_ZERO = 1e-7 diff --git a/datumaro/plugins/tf_detection_api_format/converter.py b/datumaro/plugins/tf_detection_api_format/converter.py index d2c8b6317d..c913004179 100644 --- a/datumaro/plugins/tf_detection_api_format/converter.py +++ b/datumaro/plugins/tf_detection_api_format/converter.py @@ -3,23 +3,25 @@ # # SPDX-License-Identifier: MIT -import codecs from collections import OrderedDict +import codecs import hashlib import logging as log import os import os.path as osp import string -from datumaro.components.extractor import (AnnotationType, LabelCategories) from datumaro.components.converter import Converter -from datumaro.util.image import encode_image, ByteImage -from datumaro.util.annotation_util import (max_bbox, - find_group_leader, find_instances) +from datumaro.components.extractor import AnnotationType, LabelCategories +from datumaro.util.annotation_util import ( + find_group_leader, find_instances, max_bbox, +) +from datumaro.util.image import ByteImage, encode_image from datumaro.util.mask_tools import merge_masks from datumaro.util.tf_util import import_tf as _import_tf from .format import DetectionApiPath + tf = _import_tf() diff --git a/datumaro/plugins/tf_detection_api_format/extractor.py b/datumaro/plugins/tf_detection_api_format/extractor.py index 6b62ac7dee..02d973b4c2 100644 --- a/datumaro/plugins/tf_detection_api_format/extractor.py +++ b/datumaro/plugins/tf_detection_api_format/extractor.py @@ -6,15 +6,18 @@ from collections import OrderedDict import os.path as osp import re + import numpy as np -from datumaro.components.extractor import (SourceExtractor, DatasetItem, - AnnotationType, Bbox, Mask, LabelCategories, Importer +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Importer, LabelCategories, Mask, + SourceExtractor, ) from datumaro.util.image import ByteImage, decode_image, lazy_image from datumaro.util.tf_util import import_tf as _import_tf from .format import DetectionApiPath + tf = _import_tf() diff --git a/datumaro/plugins/transforms.py b/datumaro/plugins/transforms.py index f901e567ce..7b0ad8b025 100644 --- a/datumaro/plugins/transforms.py +++ b/datumaro/plugins/transforms.py @@ -12,14 +12,15 @@ import pycocotools.mask as mask_utils -from datumaro.components.extractor import (ItemTransform, Transform, - AnnotationType, RleMask, Polygon, Bbox, Label, DEFAULT_SUBSET_NAME, - LabelCategories, MaskCategories, PointsCategories -) from datumaro.components.cli_plugin import CliPlugin -import datumaro.util.mask_tools as mask_tools -from datumaro.util import parse_str_enum_value, NOTSET +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, AnnotationType, Bbox, ItemTransform, Label, + LabelCategories, MaskCategories, PointsCategories, Polygon, RleMask, + Transform, +) +from datumaro.util import NOTSET, parse_str_enum_value from datumaro.util.annotation_util import find_group_leader, find_instances +import datumaro.util.mask_tools as mask_tools class CropCoveredSegments(ItemTransform, CliPlugin): diff --git a/datumaro/plugins/validators.py b/datumaro/plugins/validators.py index c5f936c029..c994724519 100644 --- a/datumaro/plugins/validators.py +++ b/datumaro/plugins/validators.py @@ -6,17 +6,17 @@ import numpy as np -from datumaro.components.validator import (Severity, TaskType, Validator) from datumaro.components.cli_plugin import CliPlugin -from datumaro.components.errors import (MissingLabelCategories, - MissingAnnotation, MultiLabelAnnotations, MissingAttribute, - UndefinedLabel, UndefinedAttribute, LabelDefinedButNotFound, - AttributeDefinedButNotFound, OnlyOneLabel, FewSamplesInLabel, - FewSamplesInAttribute, ImbalancedLabels, ImbalancedAttribute, - ImbalancedDistInLabel, ImbalancedDistInAttribute, - NegativeLength, InvalidValue, FarFromLabelMean, - FarFromAttrMean, OnlyOneAttributeValue) +from datumaro.components.errors import ( + AttributeDefinedButNotFound, FarFromAttrMean, FarFromLabelMean, + FewSamplesInAttribute, FewSamplesInLabel, ImbalancedAttribute, + ImbalancedDistInAttribute, ImbalancedDistInLabel, ImbalancedLabels, + InvalidValue, LabelDefinedButNotFound, MissingAnnotation, MissingAttribute, + MissingLabelCategories, MultiLabelAnnotations, NegativeLength, + OnlyOneAttributeValue, OnlyOneLabel, UndefinedAttribute, UndefinedLabel, +) from datumaro.components.extractor import AnnotationType, LabelCategories +from datumaro.components.validator import Severity, TaskType, Validator from datumaro.util import parse_str_enum_value diff --git a/datumaro/plugins/vgg_face2_format.py b/datumaro/plugins/vgg_face2_format.py index 33b4102160..d7554f3755 100644 --- a/datumaro/plugins/vgg_face2_format.py +++ b/datumaro/plugins/vgg_face2_format.py @@ -7,8 +7,10 @@ import os.path as osp from datumaro.components.converter import Converter -from datumaro.components.extractor import (AnnotationType, Bbox, DatasetItem, - Importer, Label, LabelCategories, Points, SourceExtractor) +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Importer, Label, LabelCategories, Points, + SourceExtractor, +) from datumaro.util.image import find_images diff --git a/datumaro/plugins/voc_format/converter.py b/datumaro/plugins/voc_format/converter.py index ed4b4e6a30..eef5962ba0 100644 --- a/datumaro/plugins/voc_format/converter.py +++ b/datumaro/plugins/voc_format/converter.py @@ -3,26 +3,28 @@ # # SPDX-License-Identifier: MIT -import logging as log -import os -import os.path as osp from collections import OrderedDict, defaultdict from enum import Enum, auto from itertools import chain +import logging as log +import os +import os.path as osp from lxml import etree as ET from datumaro.components.converter import Converter from datumaro.components.dataset import ItemStatus -from datumaro.components.extractor import (AnnotationType, - CompiledMask, DatasetItem, LabelCategories) +from datumaro.components.extractor import ( + AnnotationType, CompiledMask, DatasetItem, LabelCategories, +) from datumaro.util import find, str_to_bool from datumaro.util.annotation_util import make_label_id_mapping from datumaro.util.image import save_image from datumaro.util.mask_tools import paint_mask, remap_mask -from .format import (VocTask, VocPath, VocInstColormap, - parse_label_map, make_voc_label_map, make_voc_categories, write_label_map +from .format import ( + VocInstColormap, VocPath, VocTask, make_voc_categories, make_voc_label_map, + parse_label_map, write_label_map, ) diff --git a/datumaro/plugins/voc_format/extractor.py b/datumaro/plugins/voc_format/extractor.py index 4c897e7902..a2febd3900 100644 --- a/datumaro/plugins/voc_format/extractor.py +++ b/datumaro/plugins/voc_format/extractor.py @@ -6,21 +6,22 @@ from collections import defaultdict import logging as log import os.path as osp -import numpy as np + from defusedxml import ElementTree +import numpy as np -from datumaro.components.extractor import (SourceExtractor, DatasetItem, - AnnotationType, Label, Mask, Bbox, CompiledMask +from datumaro.components.extractor import ( + AnnotationType, Bbox, CompiledMask, DatasetItem, Label, Mask, + SourceExtractor, ) -from datumaro.util.os_util import dir_items from datumaro.util.image import Image, find_images -from datumaro.util.mask_tools import lazy_mask, invert_colormap +from datumaro.util.mask_tools import invert_colormap, lazy_mask +from datumaro.util.os_util import dir_items from .format import ( - VocTask, VocPath, VocInstColormap, parse_label_map, make_voc_categories + VocInstColormap, VocPath, VocTask, make_voc_categories, parse_label_map, ) - _inverse_inst_colormap = invert_colormap(VocInstColormap) class _VocExtractor(SourceExtractor): diff --git a/datumaro/plugins/voc_format/format.py b/datumaro/plugins/voc_format/format.py index 955fa609b4..6e9025c69f 100644 --- a/datumaro/plugins/voc_format/format.py +++ b/datumaro/plugins/voc_format/format.py @@ -6,10 +6,11 @@ from collections import OrderedDict from enum import Enum, auto from itertools import chain + import numpy as np -from datumaro.components.extractor import (AnnotationType, - LabelCategories, MaskCategories +from datumaro.components.extractor import ( + AnnotationType, LabelCategories, MaskCategories, ) diff --git a/datumaro/plugins/voc_format/importer.py b/datumaro/plugins/voc_format/importer.py index 0268458d40..a9918d8c02 100644 --- a/datumaro/plugins/voc_format/importer.py +++ b/datumaro/plugins/voc_format/importer.py @@ -7,7 +7,7 @@ from datumaro.components.extractor import Importer -from .format import VocTask, VocPath +from .format import VocPath, VocTask class VocImporter(Importer): @@ -20,7 +20,7 @@ class VocImporter(Importer): } def __call__(self, path, **extra_params): - from datumaro.components.project import Project # cyclic import + from datumaro.components.project import Project # cyclic import project = Project() subsets = self.find_sources(path) diff --git a/datumaro/plugins/widerface_format.py b/datumaro/plugins/widerface_format.py index a8439dc83c..829bf45026 100644 --- a/datumaro/plugins/widerface_format.py +++ b/datumaro/plugins/widerface_format.py @@ -8,8 +8,10 @@ import re from datumaro.components.converter import Converter -from datumaro.components.extractor import (AnnotationType, Bbox, DatasetItem, - Importer, Label, LabelCategories, SourceExtractor) +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Importer, Label, LabelCategories, + SourceExtractor, +) from datumaro.util import str_to_bool diff --git a/datumaro/plugins/yolo_format/converter.py b/datumaro/plugins/yolo_format/converter.py index 88a2f79255..4e768943d9 100644 --- a/datumaro/plugins/yolo_format/converter.py +++ b/datumaro/plugins/yolo_format/converter.py @@ -3,15 +3,16 @@ # # SPDX-License-Identifier: MIT +from collections import OrderedDict import logging as log import os import os.path as osp -from collections import OrderedDict from datumaro.components.converter import Converter from datumaro.components.dataset import ItemStatus -from datumaro.components.extractor import (AnnotationType, DEFAULT_SUBSET_NAME, - DatasetItem) +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, AnnotationType, DatasetItem, +) from .format import YoloPath diff --git a/datumaro/plugins/yolo_format/extractor.py b/datumaro/plugins/yolo_format/extractor.py index 4c81e929ee..82df29b0da 100644 --- a/datumaro/plugins/yolo_format/extractor.py +++ b/datumaro/plugins/yolo_format/extractor.py @@ -7,11 +7,12 @@ import os.path as osp import re -from datumaro.components.extractor import (SourceExtractor, Extractor, - DatasetItem, AnnotationType, Bbox, LabelCategories, Importer +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Extractor, Importer, LabelCategories, + SourceExtractor, ) -from datumaro.util.os_util import split_path from datumaro.util.image import Image +from datumaro.util.os_util import split_path from .format import YoloPath diff --git a/datumaro/util/annotation_util.py b/datumaro/util/annotation_util.py index a9e50306dd..aecafaee27 100644 --- a/datumaro/util/annotation_util.py +++ b/datumaro/util/annotation_util.py @@ -6,8 +6,9 @@ import numpy as np -from datumaro.components.extractor import (LabelCategories, _Shape, Mask, - AnnotationType, RleMask) +from datumaro.components.extractor import ( + AnnotationType, LabelCategories, Mask, RleMask, _Shape, +) from datumaro.util.mask_tools import mask_to_rle diff --git a/datumaro/util/command_targets.py b/datumaro/util/command_targets.py index ce79d50b75..eb9fbc9ec2 100644 --- a/datumaro/util/command_targets.py +++ b/datumaro/util/command_targets.py @@ -3,8 +3,8 @@ # # SPDX-License-Identifier: MIT -import argparse from enum import Enum, auto +import argparse from datumaro.components.project import Project from datumaro.util.image import load_image diff --git a/datumaro/util/image.py b/datumaro/util/image.py index 514d74f052..888f4b574f 100644 --- a/datumaro/util/image.py +++ b/datumaro/util/image.py @@ -4,13 +4,15 @@ # SPDX-License-Identifier: MIT from enum import Enum, auto -import importlib from io import BytesIO -from typing import Any, Callable, Iterator, Iterable, Optional, Tuple, Union +from typing import Any, Callable, Iterable, Iterator, Optional, Tuple, Union +import importlib import os import os.path as osp + import numpy as np + class _IMAGE_BACKENDS(Enum): cv2 = auto() PIL = auto() diff --git a/datumaro/util/image_cache.py b/datumaro/util/image_cache.py index 08f0258289..e37bd1358e 100644 --- a/datumaro/util/image_cache.py +++ b/datumaro/util/image_cache.py @@ -4,7 +4,6 @@ from collections import OrderedDict - _instance = None DEFAULT_CAPACITY = 2 diff --git a/datumaro/util/log_utils.py b/datumaro/util/log_utils.py index 6c8d8421e7..b4e1c5c7db 100644 --- a/datumaro/util/log_utils.py +++ b/datumaro/util/log_utils.py @@ -6,6 +6,7 @@ from contextlib import contextmanager import logging + @contextmanager def logging_disabled(max_level=logging.CRITICAL): previous_level = logging.root.manager.disable diff --git a/datumaro/util/mask_tools.py b/datumaro/util/mask_tools.py index cbe477054f..928011515b 100644 --- a/datumaro/util/mask_tools.py +++ b/datumaro/util/mask_tools.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: MIT from itertools import chain + import numpy as np from datumaro.util.image import lazy_image, load_image diff --git a/datumaro/util/os_util.py b/datumaro/util/os_util.py index 094329206a..6260f07890 100644 --- a/datumaro/util/os_util.py +++ b/datumaro/util/os_util.py @@ -8,7 +8,6 @@ import subprocess import sys - DEFAULT_MAX_DEPTH = 10 def check_instruction_set(instruction): diff --git a/datumaro/util/test_utils.py b/datumaro/util/test_utils.py index 5476188f5c..cd573243c0 100644 --- a/datumaro/util/test_utils.py +++ b/datumaro/util/test_utils.py @@ -12,13 +12,13 @@ # Use rmtree from GitPython to avoid the problem with removal of # readonly files on Windows, which Git uses extensively # It double checks if a file cannot be removed because of readonly flag - from git.util import rmtree, rmfile + from git.util import rmfile, rmtree except ImportError: from shutil import rmtree from os import remove as rmfile -from datumaro.components.extractor import AnnotationType from datumaro.components.dataset import Dataset +from datumaro.components.extractor import AnnotationType from datumaro.util import find diff --git a/datumaro/util/tf_util.py b/datumaro/util/tf_util.py index 4461984170..a0fd22e2a9 100644 --- a/datumaro/util/tf_util.py +++ b/datumaro/util/tf_util.py @@ -41,7 +41,7 @@ def import_tf(check=None): not_found = object() tf = sys.modules.get('tensorflow', not_found) if tf is None: - import tensorflow as tf # emit default error + import tensorflow as tf # emit default error elif tf is not not_found: return tf diff --git a/setup.py b/setup.py index a88778db61..dfa124e9bb 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ import os import os.path as osp import re + import setuptools # Snyk scan integration diff --git a/tests/cli/test_diff.py b/tests/cli/test_diff.py index 96bf97fa25..b80394021a 100644 --- a/tests/cli/test_diff.py +++ b/tests/cli/test_diff.py @@ -1,20 +1,19 @@ from unittest import TestCase - import os import os.path as osp import numpy as np from datumaro.cli.contexts.project.diff import DatasetDiffVisualizer +from datumaro.components.extractor import ( + AnnotationType, Bbox, Caption, DatasetItem, Label, LabelCategories, Mask, + MaskCategories, Points, PointsCategories, Polygon, PolyLine, +) from datumaro.components.operations import DistanceComparator from datumaro.components.project import Dataset -from datumaro.components.extractor import (DatasetItem, - AnnotationType, Label, Mask, Points, Polygon, - PolyLine, Bbox, Caption, - LabelCategories, MaskCategories, PointsCategories -) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir + from ..requirements import Requirements, mark_requirement diff --git a/tests/cli/test_image_zip_format.py b/tests/cli/test_image_zip_format.py index 2f7246ae07..07b742eadc 100644 --- a/tests/cli/test_image_zip_format.py +++ b/tests/cli/test_image_zip_format.py @@ -1,9 +1,10 @@ -import os -import os.path as osp from unittest import TestCase from zipfile import ZipFile +import os +import os.path as osp import numpy as np + from datumaro.cli.__main__ import main from datumaro.components.dataset import Dataset, DatasetItem from datumaro.util.test_utils import TestDir, compare_datasets diff --git a/tests/cli/test_voc_format.py b/tests/cli/test_voc_format.py index 079d4e6c98..08149f9934 100644 --- a/tests/cli/test_voc_format.py +++ b/tests/cli/test_voc_format.py @@ -1,13 +1,15 @@ -import os.path as osp from collections import OrderedDict from unittest import TestCase +import os.path as osp import numpy as np -import datumaro.plugins.voc_format.format as VOC + +from datumaro.cli.__main__ import main from datumaro.components.dataset import Dataset, DatasetItem from datumaro.components.extractor import Bbox, Label, Mask -from datumaro.cli.__main__ import main from datumaro.util.test_utils import TestDir, compare_datasets +import datumaro.plugins.voc_format.format as VOC + from ..requirements import Requirements, mark_requirement DUMMY_DATASETS_DIR = osp.join(__file__[:__file__.rfind(osp.join('tests', ''))], diff --git a/tests/cli/test_yolo_format.py b/tests/cli/test_yolo_format.py index be7590dbb2..ebe28ce2f8 100644 --- a/tests/cli/test_yolo_format.py +++ b/tests/cli/test_yolo_format.py @@ -1,15 +1,17 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.cli.__main__ import main from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (DatasetItem, - AnnotationType, Bbox) +from datumaro.components.extractor import AnnotationType, Bbox, DatasetItem from datumaro.util.test_utils import TestDir, compare_datasets import datumaro.plugins.voc_format.format as VOC + from ..requirements import Requirements, mark_requirement + def run(test, *args, expected_code=0): test.assertEqual(expected_code, main(args), str(args)) diff --git a/tests/requirements.py b/tests/requirements.py index 97e56e5577..d85f9a92a9 100644 --- a/tests/requirements.py +++ b/tests/requirements.py @@ -5,9 +5,9 @@ import typing from attr import attrs - import pytest + class DatumaroComponent: Datumaro = "datumaro" diff --git a/tests/test_RISE.py b/tests/test_RISE.py index fd756f8087..0c31031c29 100644 --- a/tests/test_RISE.py +++ b/tests/test_RISE.py @@ -2,9 +2,11 @@ from unittest import TestCase import numpy as np -from datumaro.components.extractor import Label, Bbox -from datumaro.components.launcher import Launcher + from datumaro.components.algorithms.rise import RISE +from datumaro.components.extractor import Bbox, Label +from datumaro.components.launcher import Launcher + from .requirements import Requirements, mark_requirement diff --git a/tests/test_camvid_format.py b/tests/test_camvid_format.py index 3951f8174a..05fc12630e 100644 --- a/tests/test_camvid_format.py +++ b/tests/test_camvid_format.py @@ -1,17 +1,21 @@ -import os.path as osp from collections import OrderedDict from functools import partial from unittest import TestCase +import os.path as osp import numpy as np -import datumaro.plugins.camvid_format as Camvid -from datumaro.components.extractor import (AnnotationType, DatasetItem, - Extractor, LabelCategories, Mask) + from datumaro.components.dataset import Dataset +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Extractor, LabelCategories, Mask, +) from datumaro.plugins.camvid_format import CamvidConverter, CamvidImporter from datumaro.util.image import Image -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) +import datumaro.plugins.camvid_format as Camvid + from .requirements import Requirements, mark_requirement diff --git a/tests/test_cifar_format.py b/tests/test_cifar_format.py index 1983b1a875..9b38ac0f4d 100644 --- a/tests/test_cifar_format.py +++ b/tests/test_cifar_format.py @@ -1,13 +1,16 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (AnnotationType, DatasetItem, Label, - LabelCategories) +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Label, LabelCategories, +) from datumaro.plugins.cifar_format import CifarConverter, CifarImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_cityscapes_format.py b/tests/test_cityscapes_format.py index 788e82d298..db3838d040 100644 --- a/tests/test_cityscapes_format.py +++ b/tests/test_cityscapes_format.py @@ -1,18 +1,23 @@ -import os.path as osp from collections import OrderedDict from functools import partial from unittest import TestCase +import os.path as osp import numpy as np -import datumaro.plugins.cityscapes_format as Cityscapes -from datumaro.components.extractor import (AnnotationType, DatasetItem, - Extractor, LabelCategories, Mask) + from datumaro.components.dataset import Dataset -from datumaro.plugins.cityscapes_format import (CityscapesImporter, - CityscapesConverter) +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Extractor, LabelCategories, Mask, +) +from datumaro.plugins.cityscapes_format import ( + CityscapesConverter, CityscapesImporter, +) from datumaro.util.image import Image -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) +import datumaro.plugins.cityscapes_format as Cityscapes + from .requirements import Requirements, mark_requirement DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', diff --git a/tests/test_coco_format.py b/tests/test_coco_format.py index d47514a2a4..fc6af9b024 100644 --- a/tests/test_coco_format.py +++ b/tests/test_coco_format.py @@ -1,33 +1,31 @@ from functools import partial from itertools import product +from unittest import TestCase import os import os.path as osp -from unittest import TestCase import numpy as np from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (DatasetItem, - AnnotationType, Label, Mask, Points, Polygon, Bbox, Caption, - LabelCategories, PointsCategories +from datumaro.components.extractor import ( + AnnotationType, Bbox, Caption, DatasetItem, Label, LabelCategories, Mask, + Points, PointsCategories, Polygon, ) from datumaro.plugins.coco_format.converter import ( - CocoConverter, - CocoImageInfoConverter, - CocoCaptionsConverter, - CocoInstancesConverter, - CocoPersonKeypointsConverter, - CocoLabelsConverter, - CocoPanopticConverter, - CocoStuffConverter, + CocoCaptionsConverter, CocoConverter, CocoImageInfoConverter, + CocoInstancesConverter, CocoLabelsConverter, CocoPanopticConverter, + CocoPersonKeypointsConverter, CocoStuffConverter, +) +from datumaro.plugins.coco_format.importer import ( + CocoCaptionsImporter, CocoImageInfoImporter, CocoImporter, + CocoInstancesImporter, CocoLabelsImporter, CocoPanopticImporter, + CocoPersonKeypointsImporter, CocoStuffImporter, ) -from datumaro.plugins.coco_format.importer import (CocoCaptionsImporter, - CocoImageInfoImporter, CocoImporter, CocoInstancesImporter, - CocoLabelsImporter, CocoPanopticImporter, CocoPersonKeypointsImporter, - CocoStuffImporter) from datumaro.util.image import Image -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) + from .requirements import Requirements, mark_requirement DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'coco_dataset') diff --git a/tests/test_command_targets.py b/tests/test_command_targets.py index e5e31059f0..81c7c2c374 100644 --- a/tests/test_command_targets.py +++ b/tests/test_command_targets.py @@ -1,12 +1,15 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.project import Project -from datumaro.util.command_targets import ProjectTarget, \ - ImageTarget, SourceTarget +from datumaro.util.command_targets import ( + ImageTarget, ProjectTarget, SourceTarget, +) from datumaro.util.image import save_image from datumaro.util.test_utils import TestDir + from .requirements import Requirements, mark_requirement diff --git a/tests/test_config.py b/tests/test_config.py index 2fee6b2379..a62d55fca2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,7 @@ from unittest import TestCase from datumaro.components.config import Config, DictConfig, SchemaBuilder + from .requirements import Requirements, mark_requirement diff --git a/tests/test_cvat_format.py b/tests/test_cvat_format.py index cb02b4fde7..90b599f7c2 100644 --- a/tests/test_cvat_format.py +++ b/tests/test_cvat_format.py @@ -1,19 +1,22 @@ from functools import partial +from unittest import TestCase import os import os.path as osp -from unittest import TestCase import numpy as np -from datumaro.components.project import Dataset -from datumaro.components.extractor import (DatasetItem, - AnnotationType, Points, Polygon, PolyLine, Bbox, Label, - LabelCategories, + +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Label, LabelCategories, Points, Polygon, + PolyLine, ) -from datumaro.plugins.cvat_format.extractor import CvatImporter +from datumaro.components.project import Dataset from datumaro.plugins.cvat_format.converter import CvatConverter +from datumaro.plugins.cvat_format.extractor import CvatImporter from datumaro.util.image import Image -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) + from .requirements import Requirements, mark_requirement DUMMY_IMAGE_DATASET_DIR = osp.join(osp.dirname(__file__), diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 67edf90ef1..d141d00d88 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -1,17 +1,22 @@ +from unittest import TestCase import os import os.path as osp -from unittest import TestCase import numpy as np + +from datumaro.components.dataset import ( + DEFAULT_FORMAT, Dataset, ItemStatus, eager_mode, +) from datumaro.components.dataset_filter import ( - XPathDatasetFilter, XPathAnnotationsFilter, DatasetItemEncoder) -from datumaro.components.dataset import (Dataset, DEFAULT_FORMAT, ItemStatus, - eager_mode) + DatasetItemEncoder, XPathAnnotationsFilter, XPathDatasetFilter, +) from datumaro.components.environment import Environment from datumaro.components.errors import DatumaroError, RepeatedItemError -from datumaro.components.extractor import (DEFAULT_SUBSET_NAME, Extractor, - DatasetItem, ItemTransform, Label, Mask, Points, Polygon, PolyLine, Bbox, Caption, - LabelCategories, AnnotationType, Transform) +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, AnnotationType, Bbox, Caption, DatasetItem, Extractor, + ItemTransform, Label, LabelCategories, Mask, Points, Polygon, PolyLine, + Transform, +) from datumaro.components.launcher import Launcher from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets diff --git a/tests/test_datumaro_format.py b/tests/test_datumaro_format.py index cb4c74e43c..b4318fe2f3 100644 --- a/tests/test_datumaro_format.py +++ b/tests/test_datumaro_format.py @@ -1,21 +1,23 @@ from functools import partial +from unittest import TestCase import os import os.path as osp -from unittest import TestCase import numpy as np -from datumaro.components.project import Dataset -from datumaro.components.extractor import (DatasetItem, - AnnotationType, Label, Mask, Points, Polygon, - PolyLine, Bbox, Caption, - LabelCategories, MaskCategories, PointsCategories + +from datumaro.components.extractor import ( + AnnotationType, Bbox, Caption, DatasetItem, Label, LabelCategories, Mask, + MaskCategories, Points, PointsCategories, Polygon, PolyLine, ) -from datumaro.plugins.datumaro_format.extractor import DatumaroImporter +from datumaro.components.project import Dataset from datumaro.plugins.datumaro_format.converter import DatumaroConverter -from datumaro.util.mask_tools import generate_colormap +from datumaro.plugins.datumaro_format.extractor import DatumaroImporter from datumaro.util.image import Image -from datumaro.util.test_utils import (TestDir, compare_datasets_strict, - test_save_and_load) +from datumaro.util.mask_tools import generate_colormap +from datumaro.util.test_utils import ( + TestDir, compare_datasets_strict, test_save_and_load, +) + from .requirements import Requirements, mark_requirement diff --git a/tests/test_diff.py b/tests/test_diff.py index 863f2a6faf..51f8aa32f9 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -1,10 +1,12 @@ from unittest import TestCase import numpy as np -from datumaro.components.extractor import (DatasetItem, Label, Bbox, - Caption, Mask, Points, DEFAULT_SUBSET_NAME) -from datumaro.components.project import Dataset + +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, Bbox, Caption, DatasetItem, Label, Mask, Points, +) from datumaro.components.operations import DistanceComparator, ExactComparator +from datumaro.components.project import Dataset from .requirements import Requirements, mark_requirement diff --git a/tests/test_icdar_format.py b/tests/test_icdar_format.py index 21bd8f0e0f..50a4051077 100644 --- a/tests/test_icdar_format.py +++ b/tests/test_icdar_format.py @@ -1,21 +1,26 @@ -import os.path as osp from functools import partial from unittest import TestCase +import os.path as osp import numpy as np -from datumaro.components.extractor import (Bbox, Caption, DatasetItem, Mask, - Polygon) +from datumaro.components.extractor import ( + Bbox, Caption, DatasetItem, Mask, Polygon, +) from datumaro.components.project import Dataset from datumaro.plugins.icdar_format.converter import ( IcdarTextLocalizationConverter, IcdarTextSegmentationConverter, - IcdarWordRecognitionConverter) + IcdarWordRecognitionConverter, +) from datumaro.plugins.icdar_format.extractor import ( - IcdarWordRecognitionImporter, IcdarTextLocalizationImporter, - IcdarTextSegmentationImporter) + IcdarTextLocalizationImporter, IcdarTextSegmentationImporter, + IcdarWordRecognitionImporter, +) from datumaro.util.image import Image -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) + from .requirements import Requirements, mark_requirement DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'icdar_dataset') diff --git a/tests/test_image.py b/tests/test_image.py index 75bad6a8c3..7c18bec231 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -1,10 +1,12 @@ from itertools import product -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np -import datumaro.util.image as image_module + from datumaro.util.test_utils import TestDir +import datumaro.util.image as image_module + from .requirements import Requirements, mark_requirement diff --git a/tests/test_image_dir_format.py b/tests/test_image_dir_format.py index dfc1a44d1a..cc4e04b377 100644 --- a/tests/test_image_dir_format.py +++ b/tests/test_image_dir_format.py @@ -1,13 +1,17 @@ +from unittest import TestCase import os import os.path as osp -from unittest import TestCase import numpy as np -from datumaro.components.project import Dataset + from datumaro.components.extractor import DatasetItem +from datumaro.components.project import Dataset from datumaro.plugins.image_dir_format import ImageDirConverter from datumaro.util.image import Image, save_image -from datumaro.util.test_utils import TestDir, compare_datasets, test_save_and_load +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) + from .requirements import Requirements, mark_requirement diff --git a/tests/test_image_zip_format.py b/tests/test_image_zip_format.py index a0e5b5cf36..2970dfc65d 100644 --- a/tests/test_image_zip_format.py +++ b/tests/test_image_zip_format.py @@ -1,10 +1,11 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.extractor import DatasetItem from datumaro.components.project import Dataset -from datumaro.plugins.image_zip_format import ImageZipPath, ImageZipConverter +from datumaro.plugins.image_zip_format import ImageZipConverter, ImageZipPath from datumaro.util.image import Image, save_image from datumaro.util.test_utils import TestDir, compare_datasets diff --git a/tests/test_imagenet_format.py b/tests/test_imagenet_format.py index b993f44d73..f8e009f44f 100644 --- a/tests/test_imagenet_format.py +++ b/tests/test_imagenet_format.py @@ -1,14 +1,16 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (DatasetItem, Label, - LabelCategories, AnnotationType +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Label, LabelCategories, ) from datumaro.plugins.imagenet_format import ImagenetConverter, ImagenetImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_imagenet_txt_format.py b/tests/test_imagenet_txt_format.py index b7637f991a..637bd132b1 100644 --- a/tests/test_imagenet_txt_format.py +++ b/tests/test_imagenet_txt_format.py @@ -1,15 +1,18 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (DatasetItem, Label, - LabelCategories, AnnotationType +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Label, LabelCategories, +) +from datumaro.plugins.imagenet_txt_format import ( + ImagenetTxtConverter, ImagenetTxtImporter, ) -from datumaro.plugins.imagenet_txt_format import \ - ImagenetTxtConverter, ImagenetTxtImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_images.py b/tests/test_images.py index 6bff96cbe1..1840c2b127 100644 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -1,11 +1,14 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np -from datumaro.util.test_utils import TestDir -from datumaro.util.image import (lazy_image, load_image, save_image, \ - Image, ByteImage, encode_image) + +from datumaro.util.image import ( + ByteImage, Image, encode_image, lazy_image, load_image, save_image, +) from datumaro.util.image_cache import ImageCache +from datumaro.util.test_utils import TestDir + from .requirements import Requirements, mark_requirement diff --git a/tests/test_kitti_format.py b/tests/test_kitti_format.py index e300cdc219..ac0328c8b0 100644 --- a/tests/test_kitti_format.py +++ b/tests/test_kitti_format.py @@ -1,25 +1,26 @@ -import os.path as osp from collections import OrderedDict from functools import partial from unittest import TestCase +import os.path as osp import numpy as np -from datumaro.components.extractor import (AnnotationType, Bbox, DatasetItem, - Extractor, LabelCategories, Mask) from datumaro.components.dataset import Dataset -from datumaro.plugins.kitti_format.converter import ( - KittiConverter, +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Extractor, LabelCategories, Mask, ) -from datumaro.plugins.kitti_format.format import (KittiPath, KittiTask, - make_kitti_categories, make_kitti_detection_categories, - KittiLabelMap, parse_label_map, write_label_map, +from datumaro.plugins.kitti_format.converter import KittiConverter +from datumaro.plugins.kitti_format.format import ( + KittiLabelMap, KittiPath, KittiTask, make_kitti_categories, + make_kitti_detection_categories, parse_label_map, write_label_map, +) +from datumaro.plugins.kitti_format.importer import ( + KittiDetectionImporter, KittiImporter, KittiSegmentationImporter, ) -from datumaro.plugins.kitti_format.importer import (KittiImporter, - KittiDetectionImporter, KittiSegmentationImporter) from datumaro.util.image import Image -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) from .requirements import Requirements, mark_requirement diff --git a/tests/test_labelme_format.py b/tests/test_labelme_format.py index 35faf9de16..be89add706 100644 --- a/tests/test_labelme_format.py +++ b/tests/test_labelme_format.py @@ -1,15 +1,18 @@ from functools import partial +from unittest import TestCase import os import os.path as osp -from unittest import TestCase import numpy as np + from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (DatasetItem, Bbox, Mask, Polygon) -from datumaro.plugins.labelme_format import LabelMeImporter, LabelMeConverter +from datumaro.components.extractor import Bbox, DatasetItem, Mask, Polygon +from datumaro.plugins.labelme_format import LabelMeConverter, LabelMeImporter from datumaro.util.image import Image -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) + from .requirements import Requirements, mark_requirement diff --git a/tests/test_lfw_format.py b/tests/test_lfw_format.py index 64a37731a7..b1c445f1fa 100644 --- a/tests/test_lfw_format.py +++ b/tests/test_lfw_format.py @@ -1,12 +1,14 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.dataset import Dataset from datumaro.components.extractor import DatasetItem, Label, Points from datumaro.plugins.lfw_format import LfwConverter, LfwImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_market1501_format.py b/tests/test_market1501_format.py index 17fc3afeac..26f6adf714 100644 --- a/tests/test_market1501_format.py +++ b/tests/test_market1501_format.py @@ -1,13 +1,16 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.dataset import Dataset from datumaro.components.extractor import DatasetItem -from datumaro.plugins.market1501_format import (Market1501Converter, - Market1501Importer) +from datumaro.plugins.market1501_format import ( + Market1501Converter, Market1501Importer, +) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_masks.py b/tests/test_masks.py index d17e6a2c18..5e2950b517 100644 --- a/tests/test_masks.py +++ b/tests/test_masks.py @@ -1,8 +1,10 @@ from unittest import TestCase import numpy as np -import datumaro.util.mask_tools as mask_tools + from datumaro.components.extractor import CompiledMask +import datumaro.util.mask_tools as mask_tools + from .requirements import Requirements, mark_requirement diff --git a/tests/test_mnist_csv_format.py b/tests/test_mnist_csv_format.py index 8fd258a8e5..b8ee635d30 100644 --- a/tests/test_mnist_csv_format.py +++ b/tests/test_mnist_csv_format.py @@ -1,14 +1,18 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (AnnotationType, DatasetItem, Label, - LabelCategories) -from datumaro.plugins.mnist_csv_format import (MnistCsvConverter, - MnistCsvImporter) +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Label, LabelCategories, +) +from datumaro.plugins.mnist_csv_format import ( + MnistCsvConverter, MnistCsvImporter, +) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_mnist_format.py b/tests/test_mnist_format.py index 41fe67286f..f3f219b42f 100644 --- a/tests/test_mnist_format.py +++ b/tests/test_mnist_format.py @@ -1,13 +1,16 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (AnnotationType, DatasetItem, Label, - LabelCategories) +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Label, LabelCategories, +) from datumaro.plugins.mnist_format import MnistConverter, MnistImporter from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_mot_format.py b/tests/test_mot_format.py index 176269bc7d..d133fb7991 100644 --- a/tests/test_mot_format.py +++ b/tests/test_mot_format.py @@ -1,16 +1,19 @@ from functools import partial -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (DatasetItem, - AnnotationType, Bbox, LabelCategories +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, LabelCategories, ) from datumaro.plugins.mot_format import MotSeqGtConverter, MotSeqImporter from datumaro.util.image import Image -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) + from .requirements import Requirements, mark_requirement diff --git a/tests/test_mots_format.py b/tests/test_mots_format.py index 9ad633f098..eddd909a65 100644 --- a/tests/test_mots_format.py +++ b/tests/test_mots_format.py @@ -1,14 +1,17 @@ from functools import partial -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np -from datumaro.components.extractor import DatasetItem, Mask + from datumaro.components.dataset import Dataset -from datumaro.plugins.mots_format import MotsPngConverter, MotsImporter +from datumaro.components.extractor import DatasetItem, Mask +from datumaro.plugins.mots_format import MotsImporter, MotsPngConverter from datumaro.util.image import Image -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) + from .requirements import Requirements, mark_requirement DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'mots_dataset') diff --git a/tests/test_ndr.py b/tests/test_ndr.py index 7180d1f523..731bf238d9 100644 --- a/tests/test_ndr.py +++ b/tests/test_ndr.py @@ -2,11 +2,12 @@ import numpy as np +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Label, LabelCategories, +) from datumaro.components.project import Dataset -from datumaro.components.extractor import (DatasetItem, Label, - LabelCategories, AnnotationType) - import datumaro.plugins.ndr as ndr + from .requirements import Requirements, mark_requirement diff --git a/tests/test_open_images_format.py b/tests/test_open_images_format.py index 9ba5b9a9d8..c49a804ca2 100644 --- a/tests/test_open_images_format.py +++ b/tests/test_open_images_format.py @@ -2,20 +2,24 @@ # # SPDX-License-Identifier: MIT -import os.path as osp - from unittest.case import TestCase +import os.path as osp import numpy as np from datumaro.components.dataset import Dataset -from datumaro.components.extractor import AnnotationType, DatasetItem, Label, LabelCategories -from datumaro.plugins.open_images_format import OpenImagesConverter, OpenImagesImporter +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Label, LabelCategories, +) +from datumaro.plugins.open_images_format import ( + OpenImagesConverter, OpenImagesImporter, +) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets from tests.requirements import Requirements, mark_requirement + class OpenImagesFormatTest(TestCase): @mark_requirement(Requirements.DATUM_274) def test_can_save_and_load(self): diff --git a/tests/test_ops.py b/tests/test_ops.py index 8658015dc8..d068c1719f 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -2,14 +2,19 @@ import numpy as np -from datumaro.components.extractor import (Bbox, Caption, DatasetItem, - Label, Mask, Points, Polygon, PolyLine, DEFAULT_SUBSET_NAME, - LabelCategories, PointsCategories, MaskCategories, AnnotationType) -from datumaro.components.operations import (FailedAttrVotingError, - IntersectMerge, NoMatchingAnnError, NoMatchingItemError, WrongGroupError, - compute_ann_statistics, mean_std, find_unique_images) from datumaro.components.dataset import Dataset +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, AnnotationType, Bbox, Caption, DatasetItem, Label, + LabelCategories, Mask, MaskCategories, Points, PointsCategories, Polygon, + PolyLine, +) +from datumaro.components.operations import ( + FailedAttrVotingError, IntersectMerge, NoMatchingAnnError, + NoMatchingItemError, WrongGroupError, compute_ann_statistics, + find_unique_images, mean_std, +) from datumaro.util.test_utils import compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_project.py b/tests/test_project.py index 586800745f..29faf40909 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -1,16 +1,19 @@ +from unittest import TestCase import os import os.path as osp -from unittest import TestCase import numpy as np -from datumaro.components.project import Project, Environment -from datumaro.components.config_model import Source, Model -from datumaro.components.launcher import Launcher, ModelTransform -from datumaro.components.extractor import (Extractor, DatasetItem, - Label, LabelCategories, AnnotationType) + from datumaro.components.config import Config -from datumaro.components.dataset import Dataset, DEFAULT_FORMAT +from datumaro.components.config_model import Model, Source +from datumaro.components.dataset import DEFAULT_FORMAT, Dataset +from datumaro.components.extractor import ( + AnnotationType, DatasetItem, Extractor, Label, LabelCategories, +) +from datumaro.components.launcher import Launcher, ModelTransform +from datumaro.components.project import Environment, Project from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_sampler.py b/tests/test_sampler.py index 0a587a7c67..2378fcfa5a 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -2,19 +2,18 @@ from unittest import TestCase, skipIf import csv -from datumaro.components.project import Dataset from datumaro.components.extractor import ( - DatasetItem, - Label, - LabelCategories, - AnnotationType, + AnnotationType, DatasetItem, Label, LabelCategories, ) +from datumaro.components.project import Dataset from datumaro.util.image import Image try: import pandas as pd + + from datumaro.plugins.sampler.algorithm.entropy import \ + SampleEntropy as entropy from datumaro.plugins.sampler.sampler import Sampler - from datumaro.plugins.sampler.algorithm.entropy import SampleEntropy as entropy has_libs = True except ImportError: has_libs = False diff --git a/tests/test_splitter.py b/tests/test_splitter.py index a1d7888714..82ac0851d3 100644 --- a/tests/test_splitter.py +++ b/tests/test_splitter.py @@ -1,19 +1,14 @@ from unittest import TestCase import numpy as np -from datumaro.components.project import Dataset + from datumaro.components.extractor import ( - DatasetItem, - Label, - Bbox, - Mask, - Polygon, - LabelCategories, - AnnotationType, + AnnotationType, Bbox, DatasetItem, Label, LabelCategories, Mask, Polygon, ) - -import datumaro.plugins.splitter as splitter from datumaro.components.operations import compute_ann_statistics +from datumaro.components.project import Dataset +import datumaro.plugins.splitter as splitter + from .requirements import Requirements, mark_requirement diff --git a/tests/test_tfrecord_format.py b/tests/test_tfrecord_format.py index 8b3d499cad..2a54e855e6 100644 --- a/tests/test_tfrecord_format.py +++ b/tests/test_tfrecord_format.py @@ -1,24 +1,29 @@ from functools import partial +from unittest import TestCase, skipIf import os import os.path as osp -from unittest import TestCase, skipIf import numpy as np + from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (DatasetItem, - AnnotationType, Bbox, Mask, LabelCategories +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, LabelCategories, Mask, +) +from datumaro.util.image import ByteImage, Image, encode_image +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, ) -from datumaro.util.image import Image, ByteImage, encode_image -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) from datumaro.util.tf_util import check_import + from .requirements import Requirements, mark_requirement try: - from datumaro.plugins.tf_detection_api_format.extractor import \ - TfDetectionApiExtractor, TfDetectionApiImporter - from datumaro.plugins.tf_detection_api_format.converter import \ - TfDetectionApiConverter + from datumaro.plugins.tf_detection_api_format.converter import ( + TfDetectionApiConverter, + ) + from datumaro.plugins.tf_detection_api_format.extractor import ( + TfDetectionApiExtractor, TfDetectionApiImporter, + ) import_failed = False except ImportError: import_failed = True diff --git a/tests/test_transforms.py b/tests/test_transforms.py index bf2da9000a..ea5748bd5a 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -1,15 +1,17 @@ -import logging as log from unittest import TestCase +import logging as log import numpy as np -from datumaro.components.project import Dataset -from datumaro.components.extractor import (DatasetItem, - Mask, Polygon, PolyLine, Points, Bbox, Label, - LabelCategories, MaskCategories, AnnotationType + +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Label, LabelCategories, Mask, + MaskCategories, Points, Polygon, PolyLine, ) -import datumaro.util.mask_tools as mask_tools -import datumaro.plugins.transforms as transforms +from datumaro.components.project import Dataset from datumaro.util.test_utils import compare_datasets +import datumaro.plugins.transforms as transforms +import datumaro.util.mask_tools as mask_tools + from .requirements import Requirements, mark_requirement diff --git a/tests/test_util.py b/tests/test_util.py index 53601e4228..c5048334de 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,11 +1,11 @@ +from unittest import TestCase import os import os.path as osp -from unittest import TestCase - from datumaro.util import Rollback, error_rollback -from datumaro.util.test_utils import TestDir from datumaro.util.os_util import walk +from datumaro.util.test_utils import TestDir + from .requirements import Requirements, mark_requirement diff --git a/tests/test_validator.py b/tests/test_validator.py index 22d47795ef..0e30a3a78e 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -8,18 +8,21 @@ import numpy as np from datumaro.components.dataset import Dataset, DatasetItem -from datumaro.components.errors import (MissingLabelCategories, - MissingAnnotation, MultiLabelAnnotations, MissingAttribute, - UndefinedLabel, UndefinedAttribute, LabelDefinedButNotFound, - AttributeDefinedButNotFound, OnlyOneLabel, FewSamplesInLabel, - FewSamplesInAttribute, ImbalancedLabels, ImbalancedAttribute, - ImbalancedDistInLabel, ImbalancedDistInAttribute, - NegativeLength, InvalidValue, FarFromLabelMean, - FarFromAttrMean, OnlyOneAttributeValue) +from datumaro.components.errors import ( + AttributeDefinedButNotFound, FarFromAttrMean, FarFromLabelMean, + FewSamplesInAttribute, FewSamplesInLabel, ImbalancedAttribute, + ImbalancedDistInAttribute, ImbalancedDistInLabel, ImbalancedLabels, + InvalidValue, LabelDefinedButNotFound, MissingAnnotation, MissingAttribute, + MissingLabelCategories, MultiLabelAnnotations, NegativeLength, + OnlyOneAttributeValue, OnlyOneLabel, UndefinedAttribute, UndefinedLabel, +) from datumaro.components.extractor import Bbox, Label, Mask, Polygon from datumaro.components.validator import TaskType -from datumaro.plugins.validators import (_TaskValidator, - ClassificationValidator, DetectionValidator, SegmentationValidator) +from datumaro.plugins.validators import ( + ClassificationValidator, DetectionValidator, SegmentationValidator, + _TaskValidator, +) + from .requirements import Requirements, mark_requirement diff --git a/tests/test_vgg_face2_format.py b/tests/test_vgg_face2_format.py index 9801f44d17..a8b584782e 100644 --- a/tests/test_vgg_face2_format.py +++ b/tests/test_vgg_face2_format.py @@ -1,14 +1,18 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np + from datumaro.components.dataset import Dataset -from datumaro.components.extractor import (AnnotationType, Bbox, DatasetItem, - Label, LabelCategories, Points) -from datumaro.plugins.vgg_face2_format import (VggFace2Converter, - VggFace2Importer) +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Label, LabelCategories, Points, +) +from datumaro.plugins.vgg_face2_format import ( + VggFace2Converter, VggFace2Importer, +) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_voc_format.py b/tests/test_voc_format.py index fadc6a4061..269e3def3b 100644 --- a/tests/test_voc_format.py +++ b/tests/test_voc_format.py @@ -6,26 +6,26 @@ import numpy as np -from datumaro.components.extractor import (Extractor, DatasetItem, - AnnotationType, Label, Bbox, Mask, LabelCategories, MaskCategories, +from datumaro.components.dataset import Dataset +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, Extractor, Label, LabelCategories, Mask, + MaskCategories, ) -import datumaro.plugins.voc_format.format as VOC from datumaro.plugins.voc_format.converter import ( - VocConverter, - VocClassificationConverter, - VocDetectionConverter, - VocLayoutConverter, - VocActionConverter, - VocSegmentationConverter, + VocActionConverter, VocClassificationConverter, VocConverter, + VocDetectionConverter, VocLayoutConverter, VocSegmentationConverter, +) +from datumaro.plugins.voc_format.importer import ( + VocActionImporter, VocClassificationImporter, VocDetectionImporter, + VocImporter, VocLayoutImporter, VocSegmentationImporter, ) -from datumaro.plugins.voc_format.importer import (VocActionImporter, - VocClassificationImporter, VocDetectionImporter, VocImporter, - VocLayoutImporter, VocSegmentationImporter) -from datumaro.components.dataset import Dataset from datumaro.util.image import Image from datumaro.util.mask_tools import load_mask -from datumaro.util.test_utils import (TestDir, compare_datasets, - test_save_and_load) +from datumaro.util.test_utils import ( + TestDir, compare_datasets, test_save_and_load, +) +import datumaro.plugins.voc_format.format as VOC + from .requirements import Requirements, mark_requirement diff --git a/tests/test_widerface_format.py b/tests/test_widerface_format.py index b25f987802..79996aa1c3 100644 --- a/tests/test_widerface_format.py +++ b/tests/test_widerface_format.py @@ -1,12 +1,16 @@ -import os.path as osp from unittest import TestCase +import os.path as osp import numpy as np -from datumaro.components.extractor import (Bbox, DatasetItem, Label) + from datumaro.components.dataset import Dataset -from datumaro.plugins.widerface_format import WiderFaceConverter, WiderFaceImporter +from datumaro.components.extractor import Bbox, DatasetItem, Label +from datumaro.plugins.widerface_format import ( + WiderFaceConverter, WiderFaceImporter, +) from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement diff --git a/tests/test_yolo_format.py b/tests/test_yolo_format.py index cfb4191985..3b34a7d534 100644 --- a/tests/test_yolo_format.py +++ b/tests/test_yolo_format.py @@ -1,16 +1,18 @@ +from unittest import TestCase import os import os.path as osp -from unittest import TestCase import numpy as np -from datumaro.components.extractor import (DatasetItem, - AnnotationType, Bbox, LabelCategories, -) + from datumaro.components.dataset import Dataset -from datumaro.plugins.yolo_format.extractor import YoloImporter +from datumaro.components.extractor import ( + AnnotationType, Bbox, DatasetItem, LabelCategories, +) from datumaro.plugins.yolo_format.converter import YoloConverter +from datumaro.plugins.yolo_format.extractor import YoloImporter from datumaro.util.image import Image, save_image from datumaro.util.test_utils import TestDir, compare_datasets + from .requirements import Requirements, mark_requirement From 81469eab988b1f719b7e973348dae71e87114a52 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Tue, 6 Jul 2021 13:16:53 +0300 Subject: [PATCH 3/3] Implement a workflow for checking import formatting based on isort --- .github/workflows/isort.yml | 15 +++++++++++++++ requirements.txt | 3 +++ 2 files changed, 18 insertions(+) create mode 100644 .github/workflows/isort.yml diff --git a/.github/workflows/isort.yml b/.github/workflows/isort.yml new file mode 100644 index 0000000000..22c73d4ece --- /dev/null +++ b/.github/workflows/isort.yml @@ -0,0 +1,15 @@ +name: Linter +on: pull_request +jobs: + isort: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Run checks + run: | + pip install --user -r <(grep "^isort" ./requirements.txt) + isort --check --diff . diff --git a/requirements.txt b/requirements.txt index c581bf20c5..fff66a6aa6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,8 @@ Cython>=0.27.3 # include before pycocotools opencv-python-headless>=4.1.0.25 pandas>=1.1.5 pytest>=5.3.5 + +# linters bandit>=1.7.0 +isort~=5.9 pylint>=2.7.0