From fc99b3718f87462f94cbf67314f6f1186b705886 Mon Sep 17 00:00:00 2001 From: lupeng Date: Thu, 7 Sep 2023 12:57:29 +0800 Subject: [PATCH 1/7] CocoMetric get ann_file from message --- .../datasets/base/base_coco_style_dataset.py | 7 +++++++ mmpose/evaluation/metrics/coco_metric.py | 10 +++++++++- .../test_evaluation/test_metrics/test_coco_metric.py | 12 ++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/mmpose/datasets/datasets/base/base_coco_style_dataset.py b/mmpose/datasets/datasets/base/base_coco_style_dataset.py index ec5e483a2a..d0016a7125 100644 --- a/mmpose/datasets/datasets/base/base_coco_style_dataset.py +++ b/mmpose/datasets/datasets/base/base_coco_style_dataset.py @@ -8,6 +8,7 @@ import numpy as np from mmengine.dataset import BaseDataset, force_full_init from mmengine.fileio import exists, get_local_path, load +from mmengine.logging import MessageHub from mmengine.utils import is_list_of from xtcocotools.coco import COCO @@ -112,6 +113,12 @@ def __init__(self, lazy_init=lazy_init, max_refetch=max_refetch) + if self.test_mode: + # save the ann_file into MessageHub for CocoMetric + message = MessageHub.get_current_instance() + dataset_name = self.metainfo['dataset_name'] + message.update_info_dict({f'{dataset_name}_ann_file': ann_file}) + @classmethod def _load_metainfo(cls, metainfo: dict = None) -> dict: """Collect meta information from the dictionary of meta. diff --git a/mmpose/evaluation/metrics/coco_metric.py b/mmpose/evaluation/metrics/coco_metric.py index a0ed8ef7cb..b77872e875 100644 --- a/mmpose/evaluation/metrics/coco_metric.py +++ b/mmpose/evaluation/metrics/coco_metric.py @@ -8,7 +8,7 @@ import numpy as np from mmengine.evaluator import BaseMetric from mmengine.fileio import dump, get_local_path, load -from mmengine.logging import MMLogger +from mmengine.logging import MessageHub, MMLogger from xtcocotools.coco import COCO from xtcocotools.cocoeval import COCOeval @@ -166,6 +166,14 @@ def dataset_meta(self, dataset_meta: dict) -> None: dataset_meta['num_keypoints'] = len(dataset_meta['sigmas']) self._dataset_meta = dataset_meta + if self.coco is None: + message = MessageHub.get_current_instance() + ann_file = message.get_info( + f"{dataset_meta['dataset_name']}_ann_file", None) + if ann_file is not None: + with get_local_path(ann_file) as local_path: + self.coco = COCO(local_path) + def process(self, data_batch: Sequence[dict], data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. The processed diff --git a/tests/test_evaluation/test_metrics/test_coco_metric.py b/tests/test_evaluation/test_metrics/test_coco_metric.py index 430397fd26..85c0ef56fc 100644 --- a/tests/test_evaluation/test_metrics/test_coco_metric.py +++ b/tests/test_evaluation/test_metrics/test_coco_metric.py @@ -7,8 +7,10 @@ import numpy as np from mmengine.fileio import dump, load +from mmengine.logging import MessageHub from xtcocotools.coco import COCO +from mmpose.datasets.datasets import CocoDataset from mmpose.datasets.datasets.utils import parse_pose_metainfo from mmpose.evaluation.metrics import CocoMetric @@ -664,3 +666,13 @@ def test_gt_converter(self): self.assertTrue( osp.isfile( osp.join(self.tmp_dir.name, 'test_convert.keypoints.json'))) + + def test_get_ann_file_from_dataset(self): + _ = CocoDataset(ann_file=self.ann_file_coco, test_mode=True) + metric = CocoMetric(ann_file=None) + metric.dataset_meta = self.dataset_meta_coco + self.assertIsNotNone(metric.coco) + + # clear message to avoid disturbing other tests + message = MessageHub.get_current_instance() + message.pop_info('coco_ann_file') From 487f19911273ce499ff64c800262516287b579ce Mon Sep 17 00:00:00 2001 From: lupeng Date: Thu, 7 Sep 2023 13:30:48 +0800 Subject: [PATCH 2/7] fix bug --- mmpose/datasets/datasets/base/base_coco_style_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmpose/datasets/datasets/base/base_coco_style_dataset.py b/mmpose/datasets/datasets/base/base_coco_style_dataset.py index d0016a7125..f0032aef9e 100644 --- a/mmpose/datasets/datasets/base/base_coco_style_dataset.py +++ b/mmpose/datasets/datasets/base/base_coco_style_dataset.py @@ -117,7 +117,8 @@ def __init__(self, # save the ann_file into MessageHub for CocoMetric message = MessageHub.get_current_instance() dataset_name = self.metainfo['dataset_name'] - message.update_info_dict({f'{dataset_name}_ann_file': ann_file}) + message.update_info_dict( + {f'{dataset_name}_ann_file': self.ann_file}) @classmethod def _load_metainfo(cls, metainfo: dict = None) -> dict: From 5c7ba04fe1e38981ed0a80f760493fd5d30f1b79 Mon Sep 17 00:00:00 2001 From: lupeng Date: Thu, 7 Sep 2023 15:19:22 +0800 Subject: [PATCH 3/7] align test accuracy --- mmpose/datasets/transforms/formatting.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mmpose/datasets/transforms/formatting.py b/mmpose/datasets/transforms/formatting.py index 4705916969..976697c2ea 100644 --- a/mmpose/datasets/transforms/formatting.py +++ b/mmpose/datasets/transforms/formatting.py @@ -108,7 +108,12 @@ class PackPoseInputs(BaseTransform): bbox='bboxes', bbox_score='bbox_scores', keypoints='keypoints', - keypoints_visible='keypoints_visible') + keypoints_visible='keypoints_visible', + # In CocoMetric, the area of predicted instances will be calculated + # using gt_instances.bbox_scales. To unsure correspondence with + # previous version, this key is preserved here. + bbox_scale='bbox_scales', + ) # items in `field_mapping_table` will be packed into # PoseDataSample.gt_fields and converted to Tensor. These items will be From 2784da5ff5c2d7cdb268aa8292130923fcc87ed5 Mon Sep 17 00:00:00 2001 From: lupeng Date: Thu, 7 Sep 2023 15:32:14 +0800 Subject: [PATCH 4/7] fix ut --- tests/test_evaluation/test_metrics/test_coco_metric.py | 8 ++++++++ .../test_metrics/test_coco_wholebody_metric.py | 10 ++++++++++ .../test_metrics/test_keypoint_partition_metric.py | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/tests/test_evaluation/test_metrics/test_coco_metric.py b/tests/test_evaluation/test_metrics/test_coco_metric.py index 85c0ef56fc..c935ed220e 100644 --- a/tests/test_evaluation/test_metrics/test_coco_metric.py +++ b/tests/test_evaluation/test_metrics/test_coco_metric.py @@ -23,6 +23,14 @@ def setUp(self): TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ + + # during CI on github, the unit tests for datasets will save ann_file + # into MessageHub, which will influence the unit tests for CocoMetric + msg = MessageHub.get_current_instance() + for key in msg.runtime_info: + if 'key'.endswith('ann_file'): + msg.pop_info(key) + self.tmp_dir = tempfile.TemporaryDirectory() self.ann_file_coco = 'tests/data/coco/test_coco.json' diff --git a/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py b/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py index 46e8498851..00468557b5 100644 --- a/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py +++ b/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py @@ -7,6 +7,7 @@ import numpy as np from mmengine.fileio import dump, load +from mmengine.logging import MessageHub from xtcocotools.coco import COCO from mmpose.datasets.datasets.utils import parse_pose_metainfo @@ -21,6 +22,15 @@ def setUp(self): TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ + + # during CI on github, the unit tests for datasets will save ann_file + # into MessageHub, which will influence the unit tests for + # CocoWholeBodyMetric + msg = MessageHub.get_current_instance() + for key in msg.runtime_info: + if 'key'.endswith('ann_file'): + msg.pop_info(key) + self.tmp_dir = tempfile.TemporaryDirectory() self.ann_file_coco = 'tests/data/coco/test_coco_wholebody.json' diff --git a/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py b/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py index 2b1a60c113..7c0b7fea8f 100644 --- a/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py +++ b/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py @@ -7,6 +7,7 @@ import numpy as np from mmengine.fileio import load +from mmengine.logging import MessageHub from mmengine.structures import InstanceData from xtcocotools.coco import COCO @@ -22,6 +23,14 @@ def setUp(self): TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ + + # during CI on github, the unit tests for datasets will save ann_file + # into MessageHub, which will influence the unit tests for CocoMetric + msg = MessageHub.get_current_instance() + for key in msg.runtime_info: + if 'key'.endswith('ann_file'): + msg.pop_info(key) + self.tmp_dir = tempfile.TemporaryDirectory() self.ann_file_coco = \ From 3a08eafaf6d3b24c293a5b074929721f606de763 Mon Sep 17 00:00:00 2001 From: lupeng Date: Thu, 7 Sep 2023 16:05:42 +0800 Subject: [PATCH 5/7] fix typo --- tests/test_evaluation/test_metrics/test_coco_metric.py | 2 +- .../test_evaluation/test_metrics/test_coco_wholebody_metric.py | 2 +- .../test_metrics/test_keypoint_partition_metric.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_evaluation/test_metrics/test_coco_metric.py b/tests/test_evaluation/test_metrics/test_coco_metric.py index c935ed220e..e44f31be2a 100644 --- a/tests/test_evaluation/test_metrics/test_coco_metric.py +++ b/tests/test_evaluation/test_metrics/test_coco_metric.py @@ -28,7 +28,7 @@ def setUp(self): # into MessageHub, which will influence the unit tests for CocoMetric msg = MessageHub.get_current_instance() for key in msg.runtime_info: - if 'key'.endswith('ann_file'): + if key.endswith('ann_file'): msg.pop_info(key) self.tmp_dir = tempfile.TemporaryDirectory() diff --git a/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py b/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py index 00468557b5..7d5ebbef06 100644 --- a/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py +++ b/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py @@ -28,7 +28,7 @@ def setUp(self): # CocoWholeBodyMetric msg = MessageHub.get_current_instance() for key in msg.runtime_info: - if 'key'.endswith('ann_file'): + if key.endswith('ann_file'): msg.pop_info(key) self.tmp_dir = tempfile.TemporaryDirectory() diff --git a/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py b/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py index 7c0b7fea8f..289f415f0f 100644 --- a/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py +++ b/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py @@ -28,7 +28,7 @@ def setUp(self): # into MessageHub, which will influence the unit tests for CocoMetric msg = MessageHub.get_current_instance() for key in msg.runtime_info: - if 'key'.endswith('ann_file'): + if key.endswith('ann_file'): msg.pop_info(key) self.tmp_dir = tempfile.TemporaryDirectory() From 5ef30d6ccfa9d6110faaf3eca15235736de6ae9f Mon Sep 17 00:00:00 2001 From: lupeng Date: Thu, 7 Sep 2023 16:31:12 +0800 Subject: [PATCH 6/7] fix ut --- tests/test_evaluation/test_metrics/test_coco_metric.py | 4 +--- .../test_metrics/test_coco_wholebody_metric.py | 4 +--- .../test_metrics/test_keypoint_partition_metric.py | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/test_evaluation/test_metrics/test_coco_metric.py b/tests/test_evaluation/test_metrics/test_coco_metric.py index e44f31be2a..9863f0ffe3 100644 --- a/tests/test_evaluation/test_metrics/test_coco_metric.py +++ b/tests/test_evaluation/test_metrics/test_coco_metric.py @@ -27,9 +27,7 @@ def setUp(self): # during CI on github, the unit tests for datasets will save ann_file # into MessageHub, which will influence the unit tests for CocoMetric msg = MessageHub.get_current_instance() - for key in msg.runtime_info: - if key.endswith('ann_file'): - msg.pop_info(key) + msg.runtime_info.clear() self.tmp_dir = tempfile.TemporaryDirectory() diff --git a/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py b/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py index 7d5ebbef06..fa4c04fab3 100644 --- a/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py +++ b/tests/test_evaluation/test_metrics/test_coco_wholebody_metric.py @@ -27,9 +27,7 @@ def setUp(self): # into MessageHub, which will influence the unit tests for # CocoWholeBodyMetric msg = MessageHub.get_current_instance() - for key in msg.runtime_info: - if key.endswith('ann_file'): - msg.pop_info(key) + msg.runtime_info.clear() self.tmp_dir = tempfile.TemporaryDirectory() diff --git a/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py b/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py index 289f415f0f..0d11699097 100644 --- a/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py +++ b/tests/test_evaluation/test_metrics/test_keypoint_partition_metric.py @@ -27,9 +27,7 @@ def setUp(self): # during CI on github, the unit tests for datasets will save ann_file # into MessageHub, which will influence the unit tests for CocoMetric msg = MessageHub.get_current_instance() - for key in msg.runtime_info: - if key.endswith('ann_file'): - msg.pop_info(key) + msg.runtime_info.clear() self.tmp_dir = tempfile.TemporaryDirectory() From 4973199a871fc78908b469474915464c9b04dbee Mon Sep 17 00:00:00 2001 From: lupeng Date: Fri, 8 Sep 2023 13:03:11 +0800 Subject: [PATCH 7/7] add log --- mmpose/evaluation/metrics/coco_metric.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mmpose/evaluation/metrics/coco_metric.py b/mmpose/evaluation/metrics/coco_metric.py index b77872e875..8b5e80d954 100644 --- a/mmpose/evaluation/metrics/coco_metric.py +++ b/mmpose/evaluation/metrics/coco_metric.py @@ -8,7 +8,7 @@ import numpy as np from mmengine.evaluator import BaseMetric from mmengine.fileio import dump, get_local_path, load -from mmengine.logging import MessageHub, MMLogger +from mmengine.logging import MessageHub, MMLogger, print_log from xtcocotools.coco import COCO from xtcocotools.cocoeval import COCOeval @@ -173,6 +173,10 @@ def dataset_meta(self, dataset_meta: dict) -> None: if ann_file is not None: with get_local_path(ann_file) as local_path: self.coco = COCO(local_path) + print_log( + f'CocoMetric for dataset ' + f"{dataset_meta['dataset_name']} has successfully " + f'loaded the annotation file from {ann_file}', 'current') def process(self, data_batch: Sequence[dict], data_samples: Sequence[dict]) -> None: