From e7c22316c06c9799919cbf55eeda72e563730dcb Mon Sep 17 00:00:00 2001 From: Wonju Lee Date: Thu, 15 Jun 2023 13:46:49 +0900 Subject: [PATCH 1/7] Install dvc version to 2.x (#1048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary ### How to test ### Checklist - [ ] I have added unit tests to cover my changes.​ - [ ] I have added integration tests to cover my changes.​ - [ ] I have added the description of my changes into [CHANGELOG](https://github.com/openvinotoolkit/datumaro/blob/develop/CHANGELOG.md).​ - [ ] I have updated the [documentation](https://github.com/openvinotoolkit/datumaro/tree/develop/docs) accordingly ### License - [ ] I submit _my code changes_ under the same [MIT License](https://github.com/openvinotoolkit/datumaro/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. - [ ] I have updated the license header for each file (see an example below). ```python # Copyright (C) 2023 Intel Corporation # # SPDX-License-Identifier: MIT ``` --- requirements-default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-default.txt b/requirements-default.txt index ba094413bd..8087e4ca61 100644 --- a/requirements-default.txt +++ b/requirements-default.txt @@ -1,4 +1,4 @@ -dvc>=2.7.0 +dvc>=2.7.0,<3.0.0 fsspec <= 2022.11.0; python_version < '3.8' # https://github.com/openvinotoolkit/datumaro/actions/runs/4003215621/jobs/6871114851#step:5:1647 GitPython>=3.1.18,!=3.1.25 # https://github.com/openvinotoolkit/datumaro/issues/612 openvino-telemetry>=2022.1.0 From c234574809c50cb98b9b6c69ce12bb0056bfb2d4 Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Fri, 16 Jun 2023 16:01:12 +0900 Subject: [PATCH 2/7] Replace np.append() in Validator Signed-off-by: Kim, Vinnam --- datumaro/plugins/validators.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/datumaro/plugins/validators.py b/datumaro/plugins/validators.py index d3e96a81a8..57fa582ed2 100644 --- a/datumaro/plugins/validators.py +++ b/datumaro/plugins/validators.py @@ -5,7 +5,6 @@ from copy import deepcopy import numpy as np - from datumaro.components.annotation import AnnotationType, LabelCategories from datumaro.components.cli_plugin import CliPlugin from datumaro.components.errors import ( @@ -31,6 +30,7 @@ UndefinedLabel, ) from datumaro.components.validator import Severity, TaskType, Validator + from datumaro.util import parse_str_enum_value @@ -53,7 +53,7 @@ class _TaskValidator(Validator, CliPlugin): "bins": [], "counts": [], }, - "distribution": np.array([]), + "distribution": [], } """ @@ -790,8 +790,7 @@ def generate_reports(self, stats): def _update_prop_distributions(self, curr_stats, target_stats): for prop, val in curr_stats.items(): prop_stats = target_stats[prop] - prop_dist = prop_stats["distribution"] - prop_stats["distribution"] = np.append(prop_dist, val) + prop_stats["distribution"].append(val) def _compute_prop_dist(self, label_categories, stats, update_stats_by_label): dist_by_label = stats["point_distribution_in_label"] From 1a72cffa6e2f2e74dfefd16c3ff198245d9adfd7 Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Fri, 16 Jun 2023 16:18:31 +0900 Subject: [PATCH 3/7] Let CocoBase continue even if an InvalidAnnotationError is raised Signed-off-by: Kim, Vinnam --- datumaro/plugins/data_formats/coco/base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datumaro/plugins/data_formats/coco/base.py b/datumaro/plugins/data_formats/coco/base.py index e27a2706fa..9222a19dae 100644 --- a/datumaro/plugins/data_formats/coco/base.py +++ b/datumaro/plugins/data_formats/coco/base.py @@ -10,7 +10,6 @@ import pycocotools.mask as mask_utils from attrs import define - from datumaro.components.annotation import ( AnnotationType, Bbox, @@ -34,11 +33,12 @@ ) from datumaro.components.importer import ImportContext from datumaro.components.media import Image -from datumaro.util import NOTSET, parse_json_file, take_by from datumaro.util.image import lazy_image, load_image from datumaro.util.mask_tools import bgr2index from datumaro.util.meta_file_util import has_meta_file, parse_meta_file +from datumaro.util import NOTSET, parse_json_file, take_by + from .format import CocoImporterType, CocoPath, CocoTask T = TypeVar("T") @@ -271,6 +271,8 @@ def _load_items(self, json_data): self._load_annotations( ann, img_infos[img_id], parsed_annotations=items[img_id].annotations ) + except InvalidAnnotationError as e: + log.warn(e) except Exception as e: self._ctx.error_policy.report_annotation_error( e, item_id=(img_id, self._subset) @@ -287,6 +289,8 @@ def _load_items(self, json_data): raise InvalidAnnotationError(f"Unknown image id '{img_id}'") self._load_panoptic_ann(ann, items[img_id].annotations) + except InvalidAnnotationError as e: + log.warn(e) except Exception as e: self._ctx.error_policy.report_annotation_error( e, item_id=(img_id, self._subset) From d956c2908074d37208686e04883a884d179c1096 Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Fri, 16 Jun 2023 16:20:28 +0900 Subject: [PATCH 4/7] Update CHANGELOG.md Signed-off-by: Kim, Vinnam --- CHANGELOG.md | 11 +++++++++++ docs/source/docs/release_notes.rst | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c1c9d5b4..0d202eb5fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## \[Unreleased\] +## 26/05/2023 - Release 1.3.2 +### Enhancements +- Let CocoBase continue even if an InvalidAnnotationError is raised + () + +### Bug fixes +- Install dvc version to 2.x + () +- Replace np.append() in Validator + () + ## 26/05/2023 - Release 1.3.1 ### Bug fixes - Fix Cityscapes format mis-detection diff --git a/docs/source/docs/release_notes.rst b/docs/source/docs/release_notes.rst index e3540d56aa..a74859344d 100644 --- a/docs/source/docs/release_notes.rst +++ b/docs/source/docs/release_notes.rst @@ -4,6 +4,18 @@ Release Notes .. toctree:: :maxdepth: 1 +v1.3.2 (2023.06) +---------------- + +Enhancements +^^^^^^^^^^^^ +- Let CocoBase continue even if an InvalidAnnotationError is raised + +Bug fixes +^^^^^^^^^ +- Install dvc version to 2.x +- Replace np.append() in Validator + v1.3.1 (2023.05) ---------------- From 080ee17f4dc43af23096dbfb75cf744077e43543 Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Fri, 16 Jun 2023 16:26:23 +0900 Subject: [PATCH 5/7] Version up to 1.3.2 Signed-off-by: Kim, Vinnam --- datumaro/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datumaro/version.py b/datumaro/version.py index 9c73af26be..f708a9b201 100644 --- a/datumaro/version.py +++ b/datumaro/version.py @@ -1 +1 @@ -__version__ = "1.3.1" +__version__ = "1.3.2" From c96e50a173a638c710f236cc83cbff4c323f8dde Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Fri, 16 Jun 2023 16:44:25 +0900 Subject: [PATCH 6/7] Fix isort errors Signed-off-by: Kim, Vinnam --- datumaro/plugins/data_formats/coco/base.py | 4 ++-- datumaro/plugins/validators.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datumaro/plugins/data_formats/coco/base.py b/datumaro/plugins/data_formats/coco/base.py index 9222a19dae..72cd17f427 100644 --- a/datumaro/plugins/data_formats/coco/base.py +++ b/datumaro/plugins/data_formats/coco/base.py @@ -10,6 +10,7 @@ import pycocotools.mask as mask_utils from attrs import define + from datumaro.components.annotation import ( AnnotationType, Bbox, @@ -33,12 +34,11 @@ ) from datumaro.components.importer import ImportContext from datumaro.components.media import Image +from datumaro.util import NOTSET, parse_json_file, take_by from datumaro.util.image import lazy_image, load_image from datumaro.util.mask_tools import bgr2index from datumaro.util.meta_file_util import has_meta_file, parse_meta_file -from datumaro.util import NOTSET, parse_json_file, take_by - from .format import CocoImporterType, CocoPath, CocoTask T = TypeVar("T") diff --git a/datumaro/plugins/validators.py b/datumaro/plugins/validators.py index 57fa582ed2..cad6d34174 100644 --- a/datumaro/plugins/validators.py +++ b/datumaro/plugins/validators.py @@ -5,6 +5,7 @@ from copy import deepcopy import numpy as np + from datumaro.components.annotation import AnnotationType, LabelCategories from datumaro.components.cli_plugin import CliPlugin from datumaro.components.errors import ( @@ -30,7 +31,6 @@ UndefinedLabel, ) from datumaro.components.validator import Severity, TaskType, Validator - from datumaro.util import parse_str_enum_value From fdc7ecc90d9d5e87c342f3f5c93f713a8b68a30b Mon Sep 17 00:00:00 2001 From: "Kim, Vinnam" Date: Fri, 16 Jun 2023 16:57:11 +0900 Subject: [PATCH 7/7] Fix coco test error Signed-off-by: Kim, Vinnam --- datumaro/plugins/data_formats/coco/base.py | 10 ++++------ tests/unit/test_coco_format.py | 6 +++++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/datumaro/plugins/data_formats/coco/base.py b/datumaro/plugins/data_formats/coco/base.py index 72cd17f427..49100fa6c7 100644 --- a/datumaro/plugins/data_formats/coco/base.py +++ b/datumaro/plugins/data_formats/coco/base.py @@ -266,13 +266,12 @@ def _load_items(self, json_data): try: img_id = self._parse_field(ann, "image_id", int) if img_id not in img_infos: - raise InvalidAnnotationError(f"Unknown image id '{img_id}'") + log.warn(f"Unknown image id '{img_id}'") + continue self._load_annotations( ann, img_infos[img_id], parsed_annotations=items[img_id].annotations ) - except InvalidAnnotationError as e: - log.warn(e) except Exception as e: self._ctx.error_policy.report_annotation_error( e, item_id=(img_id, self._subset) @@ -286,11 +285,10 @@ def _load_items(self, json_data): try: img_id = self._parse_field(ann, "image_id", int) if img_id not in img_infos: - raise InvalidAnnotationError(f"Unknown image id '{img_id}'") + log.warn(f"Unknown image id '{img_id}'") + continue self._load_panoptic_ann(ann, items[img_id].annotations) - except InvalidAnnotationError as e: - log.warn(e) except Exception as e: self._ctx.error_policy.report_annotation_error( e, item_id=(img_id, self._subset) diff --git a/tests/unit/test_coco_format.py b/tests/unit/test_coco_format.py index add57a2ff5..194ddff9e4 100644 --- a/tests/unit/test_coco_format.py +++ b/tests/unit/test_coco_format.py @@ -7,7 +7,7 @@ from functools import partial from io import StringIO from itertools import product -from unittest import TestCase +from unittest import TestCase, skip import numpy as np @@ -1267,6 +1267,10 @@ def test_can_report_invalid_polygon_less_than_3_points(self): self.assertIsInstance(capture.exception.__cause__, InvalidAnnotationError) self.assertIn("at least 3 (x, y) pairs", str(capture.exception.__cause__)) + @skip( + "CocoBase is changed to skip loading annotation " + "if there is no image id reference rather than raising an error." + ) @mark_requirement(Requirements.DATUM_ERROR_REPORTING) def test_can_report_invalid_image_id(self): with TestDir() as test_dir: