From ce714d281112ef4358eafc52bfb5e72565defefd Mon Sep 17 00:00:00 2001 From: Wonju Lee Date: Thu, 27 Apr 2023 13:13:32 +0900 Subject: [PATCH] Fix Mapillary Vistas data format (#977) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary ### How to test ### Checklist - [x] I have added unit tests to cover my changes.​ - [ ] I have added integration tests to cover my changes.​ - [x] I have added the description of my changes into [CHANGELOG](https://github.com/openvinotoolkit/datumaro/blob/develop/CHANGELOG.md).​ - [x] I have updated the [documentation](https://github.com/openvinotoolkit/datumaro/tree/develop/docs) accordingly ### License - [ ] I submit _my code changes_ under the same [MIT License](https://github.com/openvinotoolkit/datumaro/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. - [ ] I have updated the license header for each file (see an example below). ```python # Copyright (C) 2023 Intel Corporation # # SPDX-License-Identifier: MIT ``` --------- Co-authored-by: wonjuleee --- CHANGELOG.md | 6 +- .../data_formats/mapillary_vistas/base.py | 164 +++--- .../data_formats/mapillary_vistas/format.py | 6 +- .../data_formats/mapillary_vistas/importer.py | 27 +- .../explanation/formats/mapillary_vistas.md | 93 +-- .../config_v2.0.json => config_v1.2.json} | 0 .../mapillary_vistas_dataset/config_v2.0.json | 1 + .../dataset_with_meta_file/config_v1.2.json | 40 -- .../dataset_with_meta_file/dataset_meta.json | 4 - .../train/v1.2/instances/1.png | Bin 81 -> 0 bytes .../train/v1.2/labels/1.png | Bin 83 -> 0 bytes .../train/v1.2/labels/2.png | Bin 83 -> 0 bytes .../train/images/2.jpg | Bin .../train/v2.0 => train/v1.2}/instances/2.png | Bin .../train/v2.0 => train/v1.2}/labels/2.png | Bin .../train/v2.0 => train/v1.2}/panoptic/2.png | Bin .../v1.2/panoptic/panoptic_2018.json} | 0 .../train/v2.0/instances/2.png | Bin 0 -> 81 bytes .../train/v2.0/labels/2.png | Bin 0 -> 83 bytes .../train/v2.0/panoptic/2.png | Bin 0 -> 89 bytes .../train/v2.0/panoptic/panoptic_2020.json | 1 + .../{v2.0 => }/train/v2.0/polygons/2.json | 0 .../v1.2/config_v1.2.json | 40 -- .../v1.2/train/images/1.jpg | Bin 631 -> 0 bytes .../v1.2/train/images/2.jpg | Bin 631 -> 0 bytes .../v1.2/train/v1.2/instances/1.png | Bin 81 -> 0 bytes .../v1.2/train/v1.2/labels/1.png | Bin 83 -> 0 bytes .../v1.2/train/v1.2/labels/2.png | Bin 83 -> 0 bytes .../v1.2/val/v1.2/instances/0.png | Bin 81 -> 0 bytes .../v1.2/val/v1.2/labels/0.png | Bin 81 -> 0 bytes .../v2.0/train/images/2.jpg | Bin 631 -> 0 bytes .../v2.0/val/images/0.jpg | Bin 631 -> 0 bytes .../v2.0/val/images/1.jpg | Bin 631 -> 0 bytes .../{v1.2 => }/val/images/0.jpg | Bin .../train => val}/images/1.jpg | Bin .../val/v2.0 => val/v1.2}/instances/0.png | Bin .../val/v2.0 => val/v1.2}/instances/1.png | Bin .../{v2.0/val/v2.0 => val/v1.2}/labels/0.png | Bin .../{v2.0/val/v2.0 => val/v1.2}/labels/1.png | Bin .../val/v2.0 => val/v1.2}/panoptic/0.png | Bin .../val/v2.0 => val/v1.2}/panoptic/1.png | Bin .../v1.2/panoptic/panoptic_2018.json} | 0 .../val/v2.0/instances/0.png | Bin 0 -> 81 bytes .../val/v2.0/instances/1.png | Bin 0 -> 81 bytes .../val/v2.0/labels/0.png | Bin 0 -> 81 bytes .../val/v2.0/labels/1.png | Bin 0 -> 85 bytes .../val/v2.0/panoptic/0.png | Bin 0 -> 88 bytes .../val/v2.0/panoptic/1.png | Bin 0 -> 89 bytes .../val/v2.0/panoptic/panoptic_2020.json | 1 + .../{v2.0 => }/val/v2.0/polygons/0.json | 0 .../{v2.0 => }/val/v2.0/polygons/1.json | 0 .../test_mapillary_vistas_format.py | 548 ++++++++++++++++++ tests/unit/test_mapillary_vistas_format.py | 544 ----------------- 53 files changed, 685 insertions(+), 790 deletions(-) rename tests/assets/mapillary_vistas_dataset/{v2.0/config_v2.0.json => config_v1.2.json} (100%) create mode 100644 tests/assets/mapillary_vistas_dataset/config_v2.0.json delete mode 100644 tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/config_v1.2.json delete mode 100644 tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/dataset_meta.json delete mode 100644 tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/v1.2/instances/1.png delete mode 100644 tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/v1.2/labels/1.png delete mode 100644 tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/v1.2/labels/2.png rename tests/assets/mapillary_vistas_dataset/{dataset_with_meta_file => }/train/images/2.jpg (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/train/v2.0 => train/v1.2}/instances/2.png (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/train/v2.0 => train/v1.2}/labels/2.png (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/train/v2.0 => train/v1.2}/panoptic/2.png (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/train/v2.0/panoptic/panoptic_2020.json => train/v1.2/panoptic/panoptic_2018.json} (100%) create mode 100644 tests/assets/mapillary_vistas_dataset/train/v2.0/instances/2.png create mode 100644 tests/assets/mapillary_vistas_dataset/train/v2.0/labels/2.png create mode 100644 tests/assets/mapillary_vistas_dataset/train/v2.0/panoptic/2.png create mode 100644 tests/assets/mapillary_vistas_dataset/train/v2.0/panoptic/panoptic_2020.json rename tests/assets/mapillary_vistas_dataset/{v2.0 => }/train/v2.0/polygons/2.json (100%) delete mode 100644 tests/assets/mapillary_vistas_dataset/v1.2/config_v1.2.json delete mode 100644 tests/assets/mapillary_vistas_dataset/v1.2/train/images/1.jpg delete mode 100644 tests/assets/mapillary_vistas_dataset/v1.2/train/images/2.jpg delete mode 100644 tests/assets/mapillary_vistas_dataset/v1.2/train/v1.2/instances/1.png delete mode 100644 tests/assets/mapillary_vistas_dataset/v1.2/train/v1.2/labels/1.png delete mode 100644 tests/assets/mapillary_vistas_dataset/v1.2/train/v1.2/labels/2.png delete mode 100644 tests/assets/mapillary_vistas_dataset/v1.2/val/v1.2/instances/0.png delete mode 100644 tests/assets/mapillary_vistas_dataset/v1.2/val/v1.2/labels/0.png delete mode 100644 tests/assets/mapillary_vistas_dataset/v2.0/train/images/2.jpg delete mode 100644 tests/assets/mapillary_vistas_dataset/v2.0/val/images/0.jpg delete mode 100644 tests/assets/mapillary_vistas_dataset/v2.0/val/images/1.jpg rename tests/assets/mapillary_vistas_dataset/{v1.2 => }/val/images/0.jpg (100%) rename tests/assets/mapillary_vistas_dataset/{dataset_with_meta_file/train => val}/images/1.jpg (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/val/v2.0 => val/v1.2}/instances/0.png (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/val/v2.0 => val/v1.2}/instances/1.png (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/val/v2.0 => val/v1.2}/labels/0.png (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/val/v2.0 => val/v1.2}/labels/1.png (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/val/v2.0 => val/v1.2}/panoptic/0.png (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/val/v2.0 => val/v1.2}/panoptic/1.png (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0/val/v2.0/panoptic/panoptic_2020.json => val/v1.2/panoptic/panoptic_2018.json} (100%) create mode 100644 tests/assets/mapillary_vistas_dataset/val/v2.0/instances/0.png create mode 100644 tests/assets/mapillary_vistas_dataset/val/v2.0/instances/1.png create mode 100644 tests/assets/mapillary_vistas_dataset/val/v2.0/labels/0.png create mode 100644 tests/assets/mapillary_vistas_dataset/val/v2.0/labels/1.png create mode 100644 tests/assets/mapillary_vistas_dataset/val/v2.0/panoptic/0.png create mode 100644 tests/assets/mapillary_vistas_dataset/val/v2.0/panoptic/1.png create mode 100644 tests/assets/mapillary_vistas_dataset/val/v2.0/panoptic/panoptic_2020.json rename tests/assets/mapillary_vistas_dataset/{v2.0 => }/val/v2.0/polygons/0.json (100%) rename tests/assets/mapillary_vistas_dataset/{v2.0 => }/val/v2.0/polygons/1.json (100%) create mode 100644 tests/unit/data_formats/test_mapillary_vistas_format.py delete mode 100644 tests/unit/test_mapillary_vistas_format.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e25cceb584..9abd384de9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## \[Unreleased\] - ### New features - Add CocoRoboflowImporter () +### Enhancements + +### Bug fixes +- Fix Mapillary Vistas data format () + ## 20/04/2023 - Release 1.2.0 ### New features - Add Skill Up section to documentation diff --git a/datumaro/plugins/data_formats/mapillary_vistas/base.py b/datumaro/plugins/data_formats/mapillary_vistas/base.py index 9bc0c3116c..09e0c2dab4 100644 --- a/datumaro/plugins/data_formats/mapillary_vistas/base.py +++ b/datumaro/plugins/data_formats/mapillary_vistas/base.py @@ -1,7 +1,6 @@ -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2022-2023 Intel Corporation # # SPDX-License-Identifier: MIT -import glob import logging as log import os import os.path as osp @@ -34,8 +33,21 @@ class _MapillaryVistasBase(SubsetBase): def __init__( - self, path, task, subset=None, use_original_config=False, keep_original_category_ids=False + self, + path, + task, + subset=None, + use_original_config=False, + keep_original_category_ids=False, + format_version="v2.0", + parse_polygon=False, ): + if format_version == "v1.2" and parse_polygon is True: + raise ImportError( + f"Format version {format_version} is not available for polygons. " + "Please try with v2.0 for parsing polygons." + ) + assert osp.isdir(path), path self._path = path if subset is None: @@ -45,24 +57,25 @@ def __init__( annotations_dirs = [d for d in os.listdir(path) if d in MapillaryVistasPath.ANNOTATION_DIRS] if len(annotations_dirs) == 0: + expected_dirs = ",".join(MapillaryVistasPath.ANNOTATION_DIRS[format_version]) raise NotADirectoryError( - "Can't find annotation directory at %s. " - "Expected one of these directories: %s" - % (path, ",".join(MapillaryVistasPath.ANNOTATIONS_DIR_PATTERNS)) + f"Can't find annotation directory at {path}. " + f"Expected one of these directories: {expected_dirs}." ) elif len(annotations_dirs) > 1: + skipped_dirs = ",".join(annotations_dirs[1:]) log.warning( - "Directory(-es): %s will be skipped, dataset should contain " - "only one annotation directory" % ",".join(annotations_dirs[1:]) + f"Directory(-es): {skipped_dirs} will be skipped, dataset should " + "contain only one annotation directory" ) self._use_original_config = use_original_config - self._format_version = annotations_dirs[0] - self._annotations_dir = osp.join(path, annotations_dirs[0]) + self._format_version = format_version + self._parse_polygon = parse_polygon + self._annotations_dir = osp.join(path, format_version) self._images_dir = osp.join(path, MapillaryVistasPath.IMAGES_DIR) - self._task = task - if self._task == MapillaryVistasTask.instances: + if task == MapillaryVistasTask.instances: if has_meta_file(path): self._categories = make_mapillary_instance_categories(parse_meta_file(path)) else: @@ -75,16 +88,16 @@ def __init__( ) self._items = self._load_panoptic_items(panoptic_config) - @staticmethod - def _load_panoptic_config(path): + def _load_panoptic_config(self, path): panoptic_config_path = osp.join( - path, MapillaryVistasPath.PANOPTIC_DIR, MapillaryVistasPath.PANOPTIC_CONFIG + path, + MapillaryVistasPath.PANOPTIC_DIR, + MapillaryVistasPath.PANOPTIC_CONFIG[self._format_version], ) if not osp.isfile(panoptic_config_path): raise FileNotFoundError( - "Can't find panoptic config file: '%s' at '%s'" - % (MapillaryVistasPath.PANOPTIC_CONFIG, panoptic_config_path) + f"Can't find panoptic config file: {MapillaryVistasPath.PANOPTIC_CONFIG} at {panoptic_config_path}" ) return parse_json_file(panoptic_config_path) @@ -127,6 +140,8 @@ def _load_panoptic_items(self, config): for img in config["images"] } + polygon_dir = osp.join(self._annotations_dir, MapillaryVistasPath.POLYGON_DIR) + for item_ann in config["annotations"]: item_id = item_ann["image_id"] image = None @@ -136,13 +151,13 @@ def _load_panoptic_items(self, config): size=self._get_image_size(images_info[item_id]), ) - annotations = [] mask_path = osp.join( self._annotations_dir, MapillaryVistasPath.PANOPTIC_DIR, item_ann["file_name"] ) mask = lazy_image(mask_path, loader=self._load_pan_mask) mask = CompiledMask(instance_mask=mask) + annotations = [] for segment_info in item_ann["segments_info"]: cat_id = self._get_label_id(segment_info) segment_id = segment_info["id"] @@ -157,11 +172,24 @@ def _load_panoptic_items(self, config): ) ) + if self._parse_polygon: + polygon_path = osp.join(polygon_dir, item_id + ".json") + item_info = parse_json_file(polygon_path) + + polygons = item_info["objects"] + for polygon in polygons: + label = polygon["label"] + label_id = self._categories[AnnotationType.label].find(label)[0] + if label_id is None: + label_id = self._categories[AnnotationType.label].add(label) + + points = [int(coord) for point in polygon["polygon"] for coord in point] + annotations.append(Polygon(label=label_id, points=points)) + items[item_id] = DatasetItem( id=item_id, subset=self._subset, annotations=annotations, media=image ) - self._load_polygons(items) return items.values() def _load_instances_categories(self): @@ -180,80 +208,60 @@ def _load_instances_categories(self): def _load_instances_items(self): items = {} - instances_dir = osp.join(self._annotations_dir, MapillaryVistasPath.INSTANCES_DIR) - for instance_path in find_images(instances_dir, recursive=True): - item_id = osp.splitext(osp.relpath(instance_path, instances_dir))[0] + # class_dir = osp.join(self._annotations_dir, MapillaryVistasPath.CLASS_DIR) + # for class_path in find_images(class_dir, recursive=True): + # item_id = osp.splitext(osp.relpath(class_path, class_dir))[0] + # if item_id in items: + # continue - mask = load_image(instance_path, dtype=np.uint32) + # from PIL import Image as PILImage - annotations = [] - for uval in np.unique(mask): - label_id, instance_id = uval >> 8, uval & 255 - annotations.append( - Mask(image=self._lazy_extract_mask(mask, uval), label=label_id, id=instance_id) - ) + # class_mask = np.array(PILImage.open(class_path)) + # classes = np.unique(class_mask) - items[item_id] = DatasetItem(id=item_id, subset=self._subset, annotations=annotations) + # annotations = [] + # for label_id in classes: + # annotations.append( + # Mask(label=label_id, image=self._lazy_extract_mask(class_mask, label_id)) + # ) - class_dir = osp.join(self._annotations_dir, MapillaryVistasPath.CLASS_DIR) - for class_path in find_images(class_dir, recursive=True): - item_id = osp.splitext(osp.relpath(class_path, class_dir))[0] - if item_id in items: - continue + # items[item_id] = DatasetItem(id=item_id, subset=self._subset, annotations=annotations) - from PIL import Image as PILImage + instance_dir = osp.join(self._annotations_dir, MapillaryVistasPath.INSTANCES_DIR) + polygon_dir = osp.join(self._annotations_dir, MapillaryVistasPath.POLYGON_DIR) + for image_path in find_images(self._images_dir, recursive=True): + item_id = osp.splitext(osp.relpath(image_path, self._images_dir))[0] + image = Image.from_file(path=image_path) - class_mask = np.array(PILImage.open(class_path)) - classes = np.unique(class_mask) + instance_path = osp.join(instance_dir, item_id + MapillaryVistasPath.MASK_EXT) + mask = load_image(instance_path, dtype=np.uint32) annotations = [] - for label_id in classes: + for uval in np.unique(mask): + label_id, instance_id = uval >> 8, uval & 255 annotations.append( - Mask(label=label_id, image=self._lazy_extract_mask(class_mask, label_id)) + Mask(image=self._lazy_extract_mask(mask, uval), label=label_id, id=instance_id) ) - items[item_id] = DatasetItem(id=item_id, subset=self._subset, annotations=annotations) + if self._parse_polygon: + polygon_path = osp.join(polygon_dir, item_id + ".json") + item_info = parse_json_file(polygon_path) - for image_path in find_images(self._images_dir, recursive=True): - item_id = osp.splitext(osp.relpath(image_path, self._images_dir))[0] - image = Image.from_file(path=image_path) - if item_id in items: - items[item_id].media = image - else: - items[item_id] = DatasetItem(id=item_id, subset=self._subset, media=image) + polygons = item_info["objects"] + for polygon in polygons: + label = polygon["label"] + label_id = self._categories[AnnotationType.label].find(label)[0] + if label_id is None: + label_id = self._categories[AnnotationType.label].add(label) - self._load_polygons(items) - return items.values() - - def _load_polygons(self, items): - polygons_dir = osp.join(self._annotations_dir, MapillaryVistasPath.POLYGON_DIR) - for item_path in glob.glob(osp.join(polygons_dir, "**", "*.json"), recursive=True): - item_id = osp.splitext(osp.relpath(item_path, polygons_dir))[0] - item = items.get(item_id) - item_info = {} - item_info = parse_json_file(item_path) + points = [int(coord) for point in polygon["polygon"] for coord in point] + annotations.append(Polygon(label=label_id, points=points)) - image_size = self._get_image_size(item_info) - if image_size and item.has_image: - item.media = item.image.from_self(size=image_size) + items[item_id] = DatasetItem( + id=item_id, subset=self._subset, media=image, annotations=annotations + ) - polygons = item_info["objects"] - annotations = [] - for polygon in polygons: - label = polygon["label"] - label_id = self._categories[AnnotationType.label].find(label)[0] - if label_id is None: - label_id = self._categories[AnnotationType.label].add(label) - - points = [coord for point in polygon["polygon"] for coord in point] - annotations.append(Polygon(label=label_id, points=points)) - - if item is None: - items[item_id] = DatasetItem( - id=item_id, subset=self._subset, annotations=annotations - ) - else: - item.annotations.extend(annotations) + return items.values() @staticmethod def _get_image_size(image_info): diff --git a/datumaro/plugins/data_formats/mapillary_vistas/format.py b/datumaro/plugins/data_formats/mapillary_vistas/format.py index fd15e8470a..fe6fa23218 100644 --- a/datumaro/plugins/data_formats/mapillary_vistas/format.py +++ b/datumaro/plugins/data_formats/mapillary_vistas/format.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 Intel Corporation +# Copyright (C) 2022-2023 Intel Corporation # # SPDX-License-Identifier: MIT @@ -49,12 +49,12 @@ class MapillaryVistasPath: MASK_EXT = ".png" ANNOTATION_DIRS = { - "v1.2": [CLASS_DIR, INSTANCES_DIR], + "v1.2": [CLASS_DIR, INSTANCES_DIR, PANOPTIC_DIR], "v2.0": [CLASS_DIR, INSTANCES_DIR, PANOPTIC_DIR, POLYGON_DIR], } CONFIG_FILES = {"v1.2": "config_v1.2.json", "v2.0": "config_v2.0.json"} - PANOPTIC_CONFIG = "panoptic_2020.json" + PANOPTIC_CONFIG = {"v1.2": "panoptic_2018.json", "v2.0": "panoptic_2020.json"} CLASS_BY_DIR = { INSTANCES_DIR: MapillaryVistasTask.instances, diff --git a/datumaro/plugins/data_formats/mapillary_vistas/importer.py b/datumaro/plugins/data_formats/mapillary_vistas/importer.py index 8bb45142f9..ca2578be38 100644 --- a/datumaro/plugins/data_formats/mapillary_vistas/importer.py +++ b/datumaro/plugins/data_formats/mapillary_vistas/importer.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021 Intel Corporation +# Copyright (C) 2022-2023 Intel Corporation # # SPDX-License-Identifier: MIT import glob @@ -7,6 +7,7 @@ from datumaro.components.dataset_base import DEFAULT_SUBSET_NAME from datumaro.components.importer import Importer +from datumaro.util import str_to_bool from .base import MapillaryVistasInstancesBase, MapillaryVistasPanopticBase from .format import MapillaryVistasPath, MapillaryVistasTask @@ -21,6 +22,18 @@ class MapillaryVistasImporter(Importer): @classmethod def build_cmdline_parser(cls, **kwargs): parser = super().build_cmdline_parser(**kwargs) + parser.add_argument( + "--format-version", + default="v2.0", + type=str, + help="Use original config*.json file for your version of dataset", + ) + parser.add_argument( + "--parse-polygon", + type=str_to_bool, + default=False, + help="Use original config*.json file for your version of dataset", + ) parser.add_argument( "--use-original-config", action="store_true", @@ -39,15 +52,15 @@ def __call__(self, path, **extra_params): subsets = self.find_sources(path) if len(subsets) == 0: - raise Exception("Failed to find Mapillary Vistas dataset at '%s'" % path) + raise Exception(f"Failed to find Mapillary Vistas dataset at {path}") tasks = list(set(task for subset in subsets.values() for task in subset)) selected_task = tasks[0] if 1 < len(tasks): + task_types = ",".join(task.name for task in tasks) log.warning( - "Found potentially conflicting source types: %s" - "Only one one type will be used: %s" - % (",".join(task.name for task in tasks), selected_task.name) + f"Found potentially conflicting source types: {task_types}" + f"Only one one type will be used: {selected_task.name}" ) if selected_task == MapillaryVistasTask.instances: @@ -60,8 +73,8 @@ def __call__(self, path, **extra_params): if not has_config and not extra_params.get("use_original_config"): raise Exception( - "Failed to find config*.json at '%s'. " - "See extra args for using original config" % path + f"Failed to find config*.json at {path}. " + "See extra args for using original config." ) sources = [ diff --git a/docs/source/docs/explanation/formats/mapillary_vistas.md b/docs/source/docs/explanation/formats/mapillary_vistas.md index 9c040c509a..e06445e682 100644 --- a/docs/source/docs/explanation/formats/mapillary_vistas.md +++ b/docs/source/docs/explanation/formats/mapillary_vistas.md @@ -61,38 +61,29 @@ They differ in the number of classes, the name of the classes, supported types of annotations, and the names of the directory with annotations. So, the directory with dataset should have one of these structures: - -{{< tabpane >}} - {{< tab header="v1.2">}} -dataset -├── dataset_meta.json # a list of custom labels (optional) -├── config_v1.2.json # config file with description of classes (id, color, name) -├── -│ ├── images -│ │ ├── .jpg -│ │ ├── .jpg -│ │ ├── ... -│ └── v1.2 -│ ├── instances # directory with instance masks -│ │ └── .png -│ │ ├── .png -│ │ ├── ... -│ └── labels # directory with class masks -│ └── .png -│ ├── .png -│ ├── ... -├── -│ ├── ... -├── ... - {{< /tab >}} - {{< tab header="v2.0">}} +``` dataset +├── config_v1.2.json ├── config_v2.0.json -├── # config file with description of classes (id, color, name) +├── # config file with description of classes (id, color, name) │ ├── images │ │ ├── .jpg │ │ ├── .jpg │ │ ├── ... +│ ├── v1.2 +│ | ├── instances # directory with instance masks +│ | │ ├── .png +│ | │ ├── .png +│ | │ ├── ... +│ | ├── labels # directory with class masks +│ | │ ├── .png +│ | │ ├── .png +│ | │ ├── ... +│ | └── panoptic # directory with panoptic masks and panoptic config file +│ | ├── panoptic_2018.json # description of classes and annotations +│ | ├── .png +│ | ├── .png +│ | ├── ... │ └── v2.0 │ ├── instances # directory with instance masks │ │ ├── .png @@ -111,54 +102,10 @@ dataset │ ├── .json │ ├── .json │ ├── ... -├── - ├── ... +├── +| ├── ... ├── ... - {{< /tab >}} - {{< tab header="v1.2 w/o subsets">}} -dataset -├── config_v1.2.json # config file with description of classes (id, color, name) -├── images -│ ├── .jpg -│ ├── .jpg -│ ├── ... -└── v1.2 - ├── instances # directory with instance masks - │ └── .png - │ ├── .png - │ ├── ... - └── labels # directory with class masks - └── .png - ├── .png - ├── ... - {{< /tab >}} - {{< tab header="v2.0 w/o subsets">}} -dataset -├── config_v2.0.json -├── images -│ ├── .jpg -│ ├── .jpg -│ ├── ... -└── v2.0 - ├── instances # directory with instance masks - │ ├── .png - │ ├── .png - │ ├── ... - ├── labels # directory with class masks - │ ├── .png - │ ├── .png - │ ├── ... - ├── panoptic # directory with panoptic masks and panoptic config file - │ ├── panoptic_2020.json # description of classes and annotation objects - │ ├── .png - │ ├── .png - │ ├── ... - └── polygons # directory with description of polygons - ├── .json - ├── .json - ├── ... - {{< /tab >}} -{{< /tabpane >}} +``` To add custom classes, you can use [`dataset_meta.json`](/docs/data-formats/supported_formats.md#dataset-meta-info-file). diff --git a/tests/assets/mapillary_vistas_dataset/v2.0/config_v2.0.json b/tests/assets/mapillary_vistas_dataset/config_v1.2.json similarity index 100% rename from tests/assets/mapillary_vistas_dataset/v2.0/config_v2.0.json rename to tests/assets/mapillary_vistas_dataset/config_v1.2.json diff --git a/tests/assets/mapillary_vistas_dataset/config_v2.0.json b/tests/assets/mapillary_vistas_dataset/config_v2.0.json new file mode 100644 index 0000000000..d5698089e3 --- /dev/null +++ b/tests/assets/mapillary_vistas_dataset/config_v2.0.json @@ -0,0 +1 @@ +{"labels": [{"color": [165, 42, 42], "instances": true, "readable": "Bird", "name": "animal--bird", "evaluate": true}, {"color": [128, 128, 128], "instances": true, "readable": "Separator", "name": "construction--barrier--separator", "evaluate": true}, {"color": [119, 11, 32], "instances": true, "readable": "Bicycle", "name": "object--vehicle--bicycle", "evaluate": true}], "version": "1.1", "mapping": "public", "folder_structure": ""} \ No newline at end of file diff --git a/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/config_v1.2.json b/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/config_v1.2.json deleted file mode 100644 index 4342ea1828..0000000000 --- a/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/config_v1.2.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "labels": [ - { - "color": [ - 10, - 50, - 90 - ], - "instances": true, - "readable": "Bird", - "name": "animal--bird", - "evaluate": true - }, - { - "color": [ - 20, - 30, - 80 - ], - "instances": true, - "readable": "Curb", - "name": "construction--barrier--curb", - "evaluate": true - }, - { - "color": [ - 30, - 70, - 40 - ], - "instances": true, - "readable": "Person", - "name": "human--person", - "evaluate": true - } - ], - "version": "1.1", - "mapping": "public", - "folder_structure": "" -} diff --git a/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/dataset_meta.json b/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/dataset_meta.json deleted file mode 100644 index a50758e2a3..0000000000 --- a/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/dataset_meta.json +++ /dev/null @@ -1,4 +0,0 @@ -{ -"label_map": {"0": "animal--bird", "1": "construction--barrier--curb", "2": "human--person"}, -"segmentation_colors": [[10, 50, 90], [20, 30, 80], [30, 70, 40]] -} diff --git a/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/v1.2/instances/1.png b/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/v1.2/instances/1.png deleted file mode 100644 index 4e821fa03f78dce4ad231cdea65e399b7bdd56ee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 81 zcmeAS@N?(olHy`uVBq!ia0vp^tRO4^Bp7~7Sy~08Bs^UlLn`JZCxAc-5GJG~H83~% YFfcA+;7NA4>I_or>FVdQ&MBb@0I*UMj{pDw diff --git a/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/v1.2/labels/1.png b/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/v1.2/labels/1.png deleted file mode 100644 index 970bf5d9e94975e7100a83106e99f11edc344840..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 83 zcmeAS@N?(olHy`uVBq!ia0vp^tRTz*Bp6n(R9XTlDNh&25Dr<^gN%$oUX#Kf`&|l~ dPAGNjY+`n7Vr4rS)vySp+SAp~Wt~$(698SZ5<36@ diff --git a/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/v1.2/labels/2.png b/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/v1.2/labels/2.png deleted file mode 100644 index e5a48cbfb3a8ab9a18760410650b24b08d40850b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 83 zcmeAS@N?(olHy`uVBq!ia0vp^tRTz*Bp6n(R9XTlDNh&25Dr<^gN%$oUX#L4`&|t> dn@%WouVHq(#L9ilFf1OV+SAp~Wt~$(698m65>o&G diff --git a/tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/images/2.jpg b/tests/assets/mapillary_vistas_dataset/train/images/2.jpg similarity index 100% rename from tests/assets/mapillary_vistas_dataset/dataset_with_meta_file/train/images/2.jpg rename to tests/assets/mapillary_vistas_dataset/train/images/2.jpg diff --git a/tests/assets/mapillary_vistas_dataset/v2.0/train/v2.0/instances/2.png b/tests/assets/mapillary_vistas_dataset/train/v1.2/instances/2.png similarity index 100% rename from tests/assets/mapillary_vistas_dataset/v2.0/train/v2.0/instances/2.png rename to tests/assets/mapillary_vistas_dataset/train/v1.2/instances/2.png diff --git a/tests/assets/mapillary_vistas_dataset/v2.0/train/v2.0/labels/2.png b/tests/assets/mapillary_vistas_dataset/train/v1.2/labels/2.png similarity index 100% rename from tests/assets/mapillary_vistas_dataset/v2.0/train/v2.0/labels/2.png rename to tests/assets/mapillary_vistas_dataset/train/v1.2/labels/2.png diff --git a/tests/assets/mapillary_vistas_dataset/v2.0/train/v2.0/panoptic/2.png b/tests/assets/mapillary_vistas_dataset/train/v1.2/panoptic/2.png similarity index 100% rename from tests/assets/mapillary_vistas_dataset/v2.0/train/v2.0/panoptic/2.png rename to tests/assets/mapillary_vistas_dataset/train/v1.2/panoptic/2.png diff --git a/tests/assets/mapillary_vistas_dataset/v2.0/train/v2.0/panoptic/panoptic_2020.json b/tests/assets/mapillary_vistas_dataset/train/v1.2/panoptic/panoptic_2018.json similarity index 100% rename from tests/assets/mapillary_vistas_dataset/v2.0/train/v2.0/panoptic/panoptic_2020.json rename to tests/assets/mapillary_vistas_dataset/train/v1.2/panoptic/panoptic_2018.json diff --git a/tests/assets/mapillary_vistas_dataset/train/v2.0/instances/2.png b/tests/assets/mapillary_vistas_dataset/train/v2.0/instances/2.png new file mode 100644 index 0000000000000000000000000000000000000000..74df12285d5894d944e53ec424b653a45bbfc26d GIT binary patch literal 81 zcmeAS@N?(olHy`uVBq!ia0vp^tRO4^Bp7~7Sy~08Bs^UlLn`JZCnO}K08vUxLQ+x# abCVAPlOY3tmP2tTNUf)GAsn)-2N@ZGyas{4_Oh-n jE`oxAE-oU1g03z{SREPUvm1MhfwByqu6{1-oD!M^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGf}^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGf}I_or>FVdQ&MBb@0I*UMj{pDw diff --git a/tests/assets/mapillary_vistas_dataset/v1.2/train/v1.2/labels/1.png b/tests/assets/mapillary_vistas_dataset/v1.2/train/v1.2/labels/1.png deleted file mode 100644 index 970bf5d9e94975e7100a83106e99f11edc344840..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 83 zcmeAS@N?(olHy`uVBq!ia0vp^tRTz*Bp6n(R9XTlDNh&25Dr<^gN%$oUX#Kf`&|l~ dPAGNjY+`n7Vr4rS)vySp+SAp~Wt~$(698SZ5<36@ diff --git a/tests/assets/mapillary_vistas_dataset/v1.2/train/v1.2/labels/2.png b/tests/assets/mapillary_vistas_dataset/v1.2/train/v1.2/labels/2.png deleted file mode 100644 index e5a48cbfb3a8ab9a18760410650b24b08d40850b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 83 zcmeAS@N?(olHy`uVBq!ia0vp^tRTz*Bp6n(R9XTlDNh&25Dr<^gN%$oUX#L4`&|t> dn@%WouVHq(#L9ilFf1OV+SAp~Wt~$(698m65>o&G diff --git a/tests/assets/mapillary_vistas_dataset/v1.2/val/v1.2/instances/0.png b/tests/assets/mapillary_vistas_dataset/v1.2/val/v1.2/instances/0.png deleted file mode 100644 index f9f241e7cd89f3cf002dd34d2a56816064bf9748..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 81 zcmeAS@N?(olHy`uVBq!ia0vp^tRO4^Bp7~7Sy~08Bs^UlLn`JZrz8LYh)79DYG7{i YVPIUvz;pGc`!bMPPgg&ebxsLQ0KQ}tcK`qY diff --git a/tests/assets/mapillary_vistas_dataset/v1.2/val/v1.2/labels/0.png b/tests/assets/mapillary_vistas_dataset/v1.2/val/v1.2/labels/0.png deleted file mode 100644 index c49b4c6a7484fff94d8261bc244b141adc8c34ac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 81 zcmeAS@N?(olHy`uVBq!ia0vp^tRTz*Bp6n(R9XTl2~QWt5Dr<^gN%$oUc-bx_Of16 bGy)f;FtRWRe7bDAAEegP)z4*}Q$iB}DXS3z diff --git a/tests/assets/mapillary_vistas_dataset/v2.0/train/images/2.jpg b/tests/assets/mapillary_vistas_dataset/v2.0/train/images/2.jpg deleted file mode 100644 index 69851f88a687cf976eb1b12e53dbfee440f19bf0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 631 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGf}^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGf}^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!v`*nMGf}_=YJ$ft*5J>%Q~loCIAx!7c2k( literal 0 HcmV?d00001 diff --git a/tests/assets/mapillary_vistas_dataset/val/v2.0/labels/0.png b/tests/assets/mapillary_vistas_dataset/val/v2.0/labels/0.png new file mode 100644 index 0000000000000000000000000000000000000000..1a54e43a1b188039c103c2a7f50a0ce59b1fcfcf GIT binary patch literal 81 zcmeAS@N?(olHy`uVBq!ia0vp^tRTz*Bp6n(R9XTl2~QWt5Dr<^gN%$oUW342ds#&g YU}o6Iz-@bY(Myn8Pgg&ebxsLQ0Q?>gOaK4? literal 0 HcmV?d00001 diff --git a/tests/assets/mapillary_vistas_dataset/val/v2.0/labels/1.png b/tests/assets/mapillary_vistas_dataset/val/v2.0/labels/1.png new file mode 100644 index 0000000000000000000000000000000000000000..a9f648f38326700303b834cc47f0f172f0c89d30 GIT binary patch literal 85 zcmeAS@N?(olHy`uVBq!ia0vp^tRTz*Bp6n(R9XTl8BZ6-5Dr<^gN%$oUX#Mx|LM%` fk33{Ll?zqaePme$c73_C9;DvW)z4*}Q$iB}f148T literal 0 HcmV?d00001 diff --git a/tests/assets/mapillary_vistas_dataset/val/v2.0/panoptic/0.png b/tests/assets/mapillary_vistas_dataset/val/v2.0/panoptic/0.png new file mode 100644 index 0000000000000000000000000000000000000000..c586db44c0109a2acb4e72fc597c90578b379de8 GIT binary patch literal 88 zcmeAS@N?(olHy`uVBq!ia0vp^tRT$61SFYwH*Nw_@}4e^Asn*02OBvV7GAsn)z2N`*Ryh95v@82T) fVoIOiODK`Yyp2Kh!|mB$fwByqu6{1-oD!M