diff --git a/configs/_base_/datasets/duts.py b/configs/_base_/datasets/duts.py new file mode 100644 index 0000000000..6af6d6ccb8 --- /dev/null +++ b/configs/_base_/datasets/duts.py @@ -0,0 +1,56 @@ +# dataset settings +dataset_type = 'DUTSDataset' +data_root = 'data/DUTS' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +img_scale = (352, 352) +crop_size = (320, 320) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) + ]) +] + +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/docs/dataset_prepare.md b/docs/dataset_prepare.md index 691e63f49e..f8149ab357 100644 --- a/docs/dataset_prepare.md +++ b/docs/dataset_prepare.md @@ -108,6 +108,28 @@ mmsegmentation | | └── leftImg8bit | | | └── test | | | └── night +| ├── DUTS +│ │ ├── images +│ │ │ ├── training +│ │ │ ├── validation +│ │ ├── annotations +│ │ │ ├── training +│ │ │ ├── validation +| ├── DUT-OMRON +│ │ ├── images +│ │ │ ├── validation +│ │ ├── annotations +│ │ │ ├── validation +| ├── ECSSD +│ │ ├── images +│ │ │ ├── validation +│ │ ├── annotations +│ │ │ ├── validation +| ├── HKU-IS +│ │ ├── images +│ │ │ ├── validation +│ │ ├── annotations +│ │ │ ├── validation ``` ### Cityscapes @@ -253,3 +275,49 @@ Since we only support test models on this dataset, you may only download [the va ### Nighttime Driving Since we only support test models on this dataset, you may only download [the test set](http://data.vision.ee.ethz.ch/daid/NighttimeDriving/NighttimeDrivingTest.zip). + +### DUTS + +First，download [DUTS-TR.zip](http://saliencydetection.net/duts/download/DUTS-TR.zip) and [DUTS-TE.zip](http://saliencydetection.net/duts/download/DUTS-TE.zip) . + +To convert DUTS dataset to MMSegmentation format, you should run the following command: + +```shell +python tools/convert_datasets/duts.py /path/to/DUTS-TR.zip /path/to/DUTS-TE.zip +``` + +### DUT-OMRON + +In salient object detection (SOD), DUT-OMRON is used for evaluation. + +First，download [DUT-OMRON-image.zip](http://saliencydetection.net/dut-omron/download/DUT-OMRON-image.zip) and [DUT-OMRON-gt-pixelwise.zip.zip](http://saliencydetection.net/dut-omron/download/DUT-OMRON-gt-pixelwise.zip.zip) . + +To convert DUT-OMRON dataset to MMSegmentation format, you should run the following command: + +```shell +python tools/convert_datasets/dut_omron.py /path/to/DUT-OMRON-image.zip /path/to/DUT-OMRON-gt-pixelwise.zip.zip +``` + +### ECSSD + +In salient object detection (SOD), ECSSD is used for evaluation. + +First，download [images.zip](https://www.cse.cuhk.edu.hk/leojia/projects/hsaliency/data/ECSSD/images.zip) and [ground_truth_mask.zip](https://www.cse.cuhk.edu.hk/leojia/projects/hsaliency/data/ECSSD/ground_truth_mask.zip) . + +To convert ECSSD dataset to MMSegmentation format, you should run the following command: + +```shell +python tools/convert_datasets/ecssd.py /path/to/images.zip /path/to/ground_truth_mask.zip +``` + +### HKU-IS + +In salient object detection (SOD), HKU-IS is used for evaluation. + +First，download [HKU-IS.rar](https://sites.google.com/site/ligb86/mdfsaliency/). + +To convert HKU-IS dataset to MMSegmentation format, you should run the following command: + +```shell +python tools/convert_datasets/hku_is.py /path/to/HKU-IS.rar +``` diff --git a/docs_zh-CN/dataset_prepare.md b/docs_zh-CN/dataset_prepare.md index 80c3025de4..dd97806e04 100644 --- a/docs_zh-CN/dataset_prepare.md +++ b/docs_zh-CN/dataset_prepare.md @@ -89,6 +89,28 @@ mmsegmentation | | └── leftImg8bit | | | └── test | | | └── night +| ├── DUTS +│ │ ├── images +│ │ │ ├── training +│ │ │ ├── validation +│ │ ├── annotations +│ │ │ ├── training +│ │ │ ├── validation +| ├── DUT-OMRON +│ │ ├── images +│ │ │ ├── validation +│ │ ├── annotations +│ │ │ ├── validation +| ├── ECSSD +│ │ ├── images +│ │ │ ├── validation +│ │ ├── annotations +│ │ │ ├── validation +| ├── HKU-IS +│ │ ├── images +│ │ │ ├── validation +│ │ ├── annotations +│ │ │ ├── validation ``` ### Cityscapes @@ -195,3 +217,49 @@ python tools/convert_datasets/stare.py /path/to/stare-images.tar /path/to/labels ### Nighttime Driving 因为我们只支持在此数据集上测试模型，所以您只需下载[测试集](http://data.vision.ee.ethz.ch/daid/NighttimeDriving/NighttimeDrivingTest.zip)。 + +### DUTS + +首先，下载 [DUTS-TR.zip](http://saliencydetection.net/duts/download/DUTS-TR.zip) 和 [DUTS-TE.zip](http://saliencydetection.net/duts/download/DUTS-TE.zip) 。 + +为了将 DUTS 数据集转换成 MMSegmentation 格式，您需要运行如下命令： + +```shell +python tools/convert_datasets/duts.py /path/to/DUTS-TR.zip /path/to/DUTS-TE.zip +``` + +### DUT-OMRON + +显著性检测（SOD）任务中 DUT-OMRON 仅作为测试集。 + +首先，下载 [DUT-OMRON-image.zip](http://saliencydetection.net/dut-omron/download/DUT-OMRON-image.zip) 和 [DUT-OMRON-gt-pixelwise.zip.zip](http://saliencydetection.net/dut-omron/download/DUT-OMRON-gt-pixelwise.zip.zip) 。 + +为了将 DUT-OMRON 数据集转换成 MMSegmentation 格式，您需要运行如下命令： + +```shell +python tools/convert_datasets/dut_omron.py /path/to/DUT-OMRON-image.zip /path/to/DUT-OMRON-gt-pixelwise.zip.zip +``` + +### ECSSD + +显著性检测（SOD）任务中 ECSSD 仅作为测试集。 + +首先，下载 [images.zip](https://www.cse.cuhk.edu.hk/leojia/projects/hsaliency/data/ECSSD/images.zip) 和 [ground_truth_mask.zip](https://www.cse.cuhk.edu.hk/leojia/projects/hsaliency/data/ECSSD/ground_truth_mask.zip) 。 + +为了将 ECSSD 数据集转换成 MMSegmentation 格式，您需要运行如下命令： + +```shell +python tools/convert_datasets/ecssd.py /path/to/images.zip /path/to/ground_truth_mask.zip +``` + +### HKU-IS + +显著性检测（SOD）任务中 HKU-IS 仅作为测试集。 + +首先，下载 [HKU-IS.rar](https://sites.google.com/site/ligb86/mdfsaliency/) 。 + +为了将 HKU-IS 数据集转换成 MMSegmentation 格式，您需要运行如下命令： + +```shell +python tools/convert_datasets/hku_is.py /path/to/HKU-IS.rar +``` diff --git a/mmseg/apis/test.py b/mmseg/apis/test.py index 2b11adfdcb..3c5ebd7fc7 100644 --- a/mmseg/apis/test.py +++ b/mmseg/apis/test.py @@ -38,6 +38,7 @@ def single_gpu_test(model, efficient_test=False, opacity=0.5, pre_eval=False, + return_logit=False, format_only=False, format_args={}): """Test with single GPU by progressive mode. @@ -88,7 +89,8 @@ def single_gpu_test(model, for batch_indices, data in zip(loader_indices, data_loader): with torch.no_grad(): - result = model(return_loss=False, **data) + result = model( + return_loss=False, return_logit=return_logit, **data) if efficient_test: result = [np2tmp(_, tmpdir='.efficient_test') for _ in result] @@ -99,7 +101,8 @@ def single_gpu_test(model, if pre_eval: # TODO: adapt samples_per_gpu > 1. # only samples_per_gpu=1 valid now - result = dataset.pre_eval(result, indices=batch_indices) + result = dataset.pre_eval( + result, return_logit, indices=batch_indices) results.extend(result) @@ -142,6 +145,7 @@ def multi_gpu_test(model, gpu_collect=False, efficient_test=False, pre_eval=False, + return_logit=False, format_only=False, format_args={}): """Test model with multiple gpus by progressive mode. @@ -204,7 +208,11 @@ def multi_gpu_test(model, for batch_indices, data in zip(loader_indices, data_loader): with torch.no_grad(): - result = model(return_loss=False, rescale=True, **data) + result = model( + return_loss=False, + return_logit=return_logit, + rescale=True, + **data) if efficient_test: result = [np2tmp(_, tmpdir='.efficient_test') for _ in result] @@ -215,7 +223,8 @@ def multi_gpu_test(model, if pre_eval: # TODO: adapt samples_per_gpu > 1. # only samples_per_gpu=1 valid now - result = dataset.pre_eval(result, indices=batch_indices) + result = dataset.pre_eval( + result, return_logit, indices=batch_indices) results.extend(result) diff --git a/mmseg/core/evaluation/__init__.py b/mmseg/core/evaluation/__init__.py index 3d16d17e54..6d3bf8054b 100644 --- a/mmseg/core/evaluation/__init__.py +++ b/mmseg/core/evaluation/__init__.py @@ -2,10 +2,11 @@ from .class_names import get_classes, get_palette from .eval_hooks import DistEvalHook, EvalHook from .metrics import (eval_metrics, intersect_and_union, mean_dice, - mean_fscore, mean_iou, pre_eval_to_metrics) + mean_fscore, mean_iou, pre_eval_to_metrics, + pre_eval_to_sod_metrics, eval_sod_metrics, calc_sod_metrics) __all__ = [ 'EvalHook', 'DistEvalHook', 'mean_dice', 'mean_iou', 'mean_fscore', 'eval_metrics', 'get_classes', 'get_palette', 'pre_eval_to_metrics', - 'intersect_and_union' + 'intersect_and_union', 'calc_sod_metrics', 'eval_sod_metrics', 'pre_eval_to_sod_metrics' ] diff --git a/mmseg/core/evaluation/eval_hooks.py b/mmseg/core/evaluation/eval_hooks.py index 952db3b0b4..58a137ff57 100644 --- a/mmseg/core/evaluation/eval_hooks.py +++ b/mmseg/core/evaluation/eval_hooks.py @@ -30,9 +30,11 @@ def __init__(self, by_epoch=False, efficient_test=False, pre_eval=False, + return_logit=False, **kwargs): super().__init__(*args, by_epoch=by_epoch, **kwargs) self.pre_eval = pre_eval + self.return_logit = return_logit if efficient_test: warnings.warn( 'DeprecationWarning: ``efficient_test`` for evaluation hook ' @@ -47,7 +49,11 @@ def _do_evaluate(self, runner): from mmseg.apis import single_gpu_test results = single_gpu_test( - runner.model, self.dataloader, show=False, pre_eval=self.pre_eval) + runner.model, + self.dataloader, + show=False, + pre_eval=self.pre_eval, + return_logit=self.return_logit) runner.log_buffer.clear() runner.log_buffer.output['eval_iter_num'] = len(self.dataloader) key_score = self.evaluate(runner, results) @@ -77,9 +83,11 @@ def __init__(self, by_epoch=False, efficient_test=False, pre_eval=False, + return_logit=False, **kwargs): super().__init__(*args, by_epoch=by_epoch, **kwargs) self.pre_eval = pre_eval + self.return_logit = return_logit if efficient_test: warnings.warn( 'DeprecationWarning: ``efficient_test`` for evaluation hook ' @@ -115,7 +123,8 @@ def _do_evaluate(self, runner): self.dataloader, tmpdir=tmpdir, gpu_collect=self.gpu_collect, - pre_eval=self.pre_eval) + pre_eval=self.pre_eval, + return_logit=self.return_logit) runner.log_buffer.clear() diff --git a/mmseg/core/evaluation/metrics.py b/mmseg/core/evaluation/metrics.py index b83a798ea9..7bd3078f2d 100644 --- a/mmseg/core/evaluation/metrics.py +++ b/mmseg/core/evaluation/metrics.py @@ -18,11 +18,29 @@ def f_score(precision, recall, beta=1): Returns: [torch.tensor]: The f-score value. """ - score = (1 + beta**2) * (precision * recall) / ( - (beta**2 * precision) + recall) + score = (1 + beta ** 2) * (precision * recall) / ( + (beta ** 2 * precision) + recall) return score +def calc_mae(pred_label, label): + return torch.mean(torch.abs(pred_label - label)) + + +def calc_adaptive_fm(pred_label, label, beta=0.3): + adaptive_threshold = min(2 * pred_label.mean(), 1.) + binary_pred_label = pred_label >= adaptive_threshold + area_intersection = binary_pred_label[label].sum() + if area_intersection == 0: + adaptive_fm = 0 + else: + precision = area_intersection / torch.count_nonzero(binary_pred_label) + recall = area_intersection / torch.count_nonzero(label) + adaptive_fm = (1 + beta) * precision * recall / ( + beta * precision + recall) + return adaptive_fm + + def intersect_and_union(pred_label, label, num_classes, @@ -112,10 +130,10 @@ def total_intersect_and_union(results, ndarray: The prediction histogram on all classes. ndarray: The ground truth histogram on all classes. """ - total_area_intersect = torch.zeros((num_classes, ), dtype=torch.float64) - total_area_union = torch.zeros((num_classes, ), dtype=torch.float64) - total_area_pred_label = torch.zeros((num_classes, ), dtype=torch.float64) - total_area_label = torch.zeros((num_classes, ), dtype=torch.float64) + total_area_intersect = torch.zeros((num_classes,), dtype=torch.float64) + total_area_union = torch.zeros((num_classes,), dtype=torch.float64) + total_area_pred_label = torch.zeros((num_classes,), dtype=torch.float64) + total_area_label = torch.zeros((num_classes,), dtype=torch.float64) for result, gt_seg_map in zip(results, gt_seg_maps): area_intersect, area_union, area_pred_label, area_label = \ intersect_and_union( @@ -126,7 +144,7 @@ def total_intersect_and_union(results, total_area_pred_label += area_pred_label total_area_label += area_label return total_area_intersect, total_area_union, total_area_pred_label, \ - total_area_label + total_area_label def mean_iou(results, @@ -282,9 +300,9 @@ def eval_metrics(results, """ total_area_intersect, total_area_union, total_area_pred_label, \ - total_area_label = total_intersect_and_union( - results, gt_seg_maps, num_classes, ignore_index, label_map, - reduce_zero_label) + total_area_label = total_intersect_and_union( + results, gt_seg_maps, num_classes, ignore_index, label_map, + reduce_zero_label) ret_metrics = total_area_to_metrics(total_area_intersect, total_area_union, total_area_pred_label, total_area_label, metrics, nan_to_num, @@ -293,6 +311,67 @@ def eval_metrics(results, return ret_metrics +def calc_sod_metrics(pred_label, label): + if isinstance(pred_label, str): + pred_label = torch.from_numpy(np.load(pred_label)) + else: + pred_label = torch.from_numpy((pred_label)) + + if isinstance(label, str): + label = torch.from_numpy( + mmcv.imread(label, flag='unchanged', backend='pillow')) + else: + label = torch.from_numpy(label) + + pred_label = pred_label.float() + if pred_label.max() != pred_label.min(): + pred_label = (pred_label - pred_label.min()) / ( + pred_label.max() - pred_label.min()) + + mae = calc_mae(pred_label, label) + adaptive_fm = calc_adaptive_fm(pred_label, label) + + return mae, adaptive_fm + + +def pre_eval_to_sod_metrics(pre_eval_results, nan_to_num=None): + pre_eval_results = tuple(zip(*pre_eval_results)) + assert len(pre_eval_results) == 2 + + mae = sum(pre_eval_results[0]) / len(pre_eval_results[0]) + adp_fm = sum(pre_eval_results[1]) / len(pre_eval_results[1]) + + ret_metrics = OrderedDict({'MAE': mae.numpy(), 'adpFm': adp_fm.numpy()}) + if nan_to_num is not None: + ret_metrics = OrderedDict({ + metric: np.nan_to_num(metric_value, nan=nan_to_num) + for metric, metric_value in ret_metrics.items() + }) + return ret_metrics + + +def eval_sod_metrics(results, gt_seg_maps, nan_to_num=None): + maes = [] + adp_fms = [] + + for result, gt_seg_map in zip(results, gt_seg_maps): + mae, adp_fm = calc_sod_metrics(result, gt_seg_map) + maes.append(mae) + adp_fms.append(adp_fm) + + ret_metrics = OrderedDict({ + 'MAE': np.mean(maes), + 'adpFm': np.mean(adp_fms) + }) + + if nan_to_num is not None: + ret_metrics = OrderedDict({ + metric: np.nan_to_num(metric_value, nan=nan_to_num) + for metric, metric_value in ret_metrics.items() + }) + return ret_metrics + + def pre_eval_to_metrics(pre_eval_results, metrics=['mIoU'], nan_to_num=None, @@ -370,7 +449,7 @@ def total_area_to_metrics(total_area_intersect, ret_metrics['Acc'] = acc elif metric == 'mDice': dice = 2 * total_area_intersect / ( - total_area_pred_label + total_area_label) + total_area_pred_label + total_area_label) acc = total_area_intersect / total_area_label ret_metrics['Dice'] = dice ret_metrics['Acc'] = acc diff --git a/mmseg/datasets/__init__.py b/mmseg/datasets/__init__.py index 4b8e124cf8..3e0d746dee 100644 --- a/mmseg/datasets/__init__.py +++ b/mmseg/datasets/__init__.py @@ -8,11 +8,16 @@ from .dark_zurich import DarkZurichDataset from .dataset_wrappers import ConcatDataset, RepeatDataset from .drive import DRIVEDataset +from .dut_omron import DUTOMRONDataset +from .duts import DUTSDataset +from .ecssd import ECSSDDataset +from .hku_is import HKUISDataset from .hrf import HRFDataset from .night_driving import NightDrivingDataset from .pascal_context import PascalContextDataset, PascalContextDataset59 from .stare import STAREDataset from .voc import PascalVOCDataset +from .sod_custom import SODCustomDataset __all__ = [ 'CustomDataset', 'build_dataloader', 'ConcatDataset', 'RepeatDataset', @@ -20,5 +25,6 @@ 'PascalVOCDataset', 'ADE20KDataset', 'PascalContextDataset', 'PascalContextDataset59', 'ChaseDB1Dataset', 'DRIVEDataset', 'HRFDataset', 'STAREDataset', 'DarkZurichDataset', 'NightDrivingDataset', - 'COCOStuffDataset' + 'COCOStuffDataset', 'DUTSDataset', 'DUTOMRONDataset', 'ECSSDDataset', + 'HKUISDataset', 'SODCustomDataset' ] diff --git a/mmseg/datasets/custom.py b/mmseg/datasets/custom.py index 23b347d34b..b2d0d63c4a 100644 --- a/mmseg/datasets/custom.py +++ b/mmseg/datasets/custom.py @@ -5,6 +5,7 @@ import mmcv import numpy as np +import torch.nn.functional as F from mmcv.utils import print_log from prettytable import PrettyTable from torch.utils.data import Dataset @@ -262,7 +263,7 @@ def get_gt_seg_maps(self, efficient_test=None): self.gt_seg_map_loader(results) yield results['gt_semantic_seg'] - def pre_eval(self, preds, indices): + def pre_eval(self, preds, indices, return_logit=False): """Collect eval result from each iteration. Args: @@ -284,6 +285,14 @@ def pre_eval(self, preds, indices): pre_eval_results = [] for pred, index in zip(preds, indices): + if return_logit: + if pred.shape[0] >= 2: + pred = F.softmax(pred, dim=0) + pred = pred.argmax(dim=0) + else: + pred = F.sigmoid(pred) + pred = pred.squeeze(0) + pred = (pred > 0.5).int() seg_map = self.get_gt_seg_map_by_idx(index) pre_eval_results.append( intersect_and_union(pred, seg_map, len(self.CLASSES), @@ -358,6 +367,7 @@ def get_palette_for_custom_classes(self, class_names, palette=None): def evaluate(self, results, metric='mIoU', + return_logit=False, logger=None, gt_seg_maps=None, **kwargs): diff --git a/mmseg/datasets/dut_omron.py b/mmseg/datasets/dut_omron.py new file mode 100644 index 0000000000..a2d44ed442 --- /dev/null +++ b/mmseg/datasets/dut_omron.py @@ -0,0 +1,27 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp + +from .sod_custom import SODCustomDataset +from .builder import DATASETS + + +@DATASETS.register_module() +class DUTOMRONDataset(SODCustomDataset): + """DUT-OMRON dataset. + + In saliency map annotation for DUT-OMRON, 0 stands for background. + ``reduce_zero_label`` is fixed to False. The ``img_suffix`` is fixed to + '.png' and ``seg_map_suffix`` is fixed to '.png'. + """ + + CLASSES = ('background', 'foreground') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(DUTOMRONDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/mmseg/datasets/duts.py b/mmseg/datasets/duts.py new file mode 100644 index 0000000000..f44b93c8f3 --- /dev/null +++ b/mmseg/datasets/duts.py @@ -0,0 +1,27 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp + +from .builder import DATASETS +from .sod_custom import SODCustomDataset + + +@DATASETS.register_module() +class DUTSDataset(SODCustomDataset): + """DUTS dataset. + + In saliency map annotation for DUTS, 0 stands for background. + ``reduce_zero_label`` is fixed to False. The ``img_suffix`` is fixed to + '.png' and ``seg_map_suffix`` is fixed to '.png'. + """ + + CLASSES = ('background', 'foreground') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(DUTSDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/mmseg/datasets/ecssd.py b/mmseg/datasets/ecssd.py new file mode 100644 index 0000000000..45a56bdcbd --- /dev/null +++ b/mmseg/datasets/ecssd.py @@ -0,0 +1,27 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp + +from .builder import DATASETS +from .sod_custom import SODCustomDataset + + +@DATASETS.register_module() +class ECSSDDataset(SODCustomDataset): + """ECSSD dataset. + + In saliency map annotation for ECSSD, 0 stands for background. + ``reduce_zero_label`` is fixed to False. The ``img_suffix`` is fixed to + '.png' and ``seg_map_suffix`` is fixed to '.png'. + """ + + CLASSES = ('background', 'foreground') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(ECSSDDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/mmseg/datasets/hku_is.py b/mmseg/datasets/hku_is.py new file mode 100644 index 0000000000..17dc56bb78 --- /dev/null +++ b/mmseg/datasets/hku_is.py @@ -0,0 +1,27 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp + +from .builder import DATASETS +from .sod_custom import SODCustomDataset + + +@DATASETS.register_module() +class HKUISDataset(SODCustomDataset): + """HKU-IS dataset. + + In saliency map annotation for HKU-IS, 0 stands for background. + ``reduce_zero_label`` is fixed to False. The ``img_suffix`` is fixed to + '.png' and ``seg_map_suffix`` is fixed to '.png'. + """ + + CLASSES = ('background', 'foreground') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(HKUISDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/mmseg/datasets/sod_custom.py b/mmseg/datasets/sod_custom.py new file mode 100644 index 0000000000..fdf01f3674 --- /dev/null +++ b/mmseg/datasets/sod_custom.py @@ -0,0 +1,110 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from collections import OrderedDict + +import mmcv +import numpy as np +import torch.nn.functional as F +from mmcv.utils import print_log +from prettytable import PrettyTable + +from mmseg.core import (calc_sod_metrics, eval_sod_metrics, + pre_eval_to_sod_metrics) +from . import CustomDataset +from .builder import DATASETS + + +@DATASETS.register_module() +class SODCustomDataset(CustomDataset): + CLASSES = None + + PALETTE = None + + def __init__(self, **kwargs): + super(SODCustomDataset, self).__init__(**kwargs) + + def pre_eval(self, preds, indices, return_logit=False): + """Collect eval result from each iteration. + + Args: + preds (list[torch.Tensor] | torch.Tensor): the segmentation logit + after argmax, shape (N, H, W). + indices (list[int] | int): the prediction related ground truth + indices. + + Returns: + list[torch.Tensor]: (area_intersect, area_union, area_prediction, + area_ground_truth). + """ + # In order to compat with batch inference + if not isinstance(indices, list): + indices = [indices] + if not isinstance(preds, list): + preds = [preds] + + pre_eval_results = [] + + for pred, index in zip(preds, indices): + if return_logit: + if pred.shape[0] >= 2: + pred = F.softmax(pred, dim=0) + pred = pred[1] + else: + pred = F.sigmoid(pred) + pred = pred.squeeze(0) + seg_map = self.get_gt_seg_map_by_idx(index) + pre_eval_results.append(calc_sod_metrics(pred, seg_map)) + + return pre_eval_results + + def evaluate(self, + results, + logger=None, + return_logit=False, + gt_seg_maps=None, + **kwargs): + """Evaluate the dataset. + + Args: + results (list[tuple[torch.Tensor]] | list[str]): per image pre_eval + results or predict segmentation map for computing evaluation + metric. + metric (str | list[str]): Metrics to be evaluated. 'mIoU', + 'mDice' and 'mFscore' are supported. + logger (logging.Logger | None | str): Logger used for printing + related information during evaluation. Default: None. + gt_seg_maps (generator[ndarray]): Custom gt seg maps as input, + used in ConcatDataset + + Returns: + dict[str, float]: Default metrics. + """ + + eval_results = {} + # test a list of files + if mmcv.is_list_of(results, np.ndarray) or mmcv.is_list_of( + results, str): + if gt_seg_maps is None: + gt_seg_maps = self.get_gt_seg_maps() + ret_metrics = eval_sod_metrics(results, gt_seg_maps) + # test a list of pre_eval_results + else: + ret_metrics = pre_eval_to_sod_metrics(results) + + # summary table + ret_metrics_summary = OrderedDict({ + ret_metric: np.round(np.nanmean(ret_metric_value) * 100, 2) + for ret_metric, ret_metric_value in ret_metrics.items() + }) + + summary_table_data = PrettyTable() + for key, val in ret_metrics_summary.items(): + summary_table_data.add_column(key, [val]) + + print_log('Summary:', logger) + print_log('\n' + summary_table_data.get_string(), logger=logger) + + # each metric dict + for key, value in ret_metrics_summary.items(): + eval_results[key] = value / 100.0 + + return eval_results diff --git a/mmseg/models/losses/cross_entropy_loss.py b/mmseg/models/losses/cross_entropy_loss.py index ee489a888f..ae33fe550c 100644 --- a/mmseg/models/losses/cross_entropy_loss.py +++ b/mmseg/models/losses/cross_entropy_loss.py @@ -83,8 +83,11 @@ def binary_cross_entropy(pred, pred.dim() == 4 and label.dim() == 3), \ 'Only pred shape [N, C], label shape [N] or pred shape [N, C, ' \ 'H, W], label shape [N, H, W] are supported' - label, weight = _expand_onehot_labels(label, weight, pred.shape, - ignore_index) + if pred.shape[1] == 1: + pred = pred.squeeze(1) + else: + label, weight = _expand_onehot_labels(label, weight, pred.shape, + ignore_index) # weighted element-wise losses if weight is not None: diff --git a/mmseg/models/segmentors/encoder_decoder.py b/mmseg/models/segmentors/encoder_decoder.py index 72467b4690..53d880046e 100644 --- a/mmseg/models/segmentors/encoder_decoder.py +++ b/mmseg/models/segmentors/encoder_decoder.py @@ -216,7 +216,7 @@ def whole_inference(self, img, img_meta, rescale): return seg_logit - def inference(self, img, img_meta, rescale): + def inference(self, img, img_meta, rescale, return_logit): """Inference with slide/whole style. Args: @@ -239,7 +239,13 @@ def inference(self, img, img_meta, rescale): seg_logit = self.slide_inference(img, img_meta, rescale) else: seg_logit = self.whole_inference(img, img_meta, rescale) - output = F.softmax(seg_logit, dim=1) + if return_logit: + output = seg_logit + else: + if seg_logit.shape[1] >= 2: + output = F.softmax(seg_logit, dim=1) + else: + output = F.sigmoid(seg_logit) flip = img_meta[0]['flip'] if flip: flip_direction = img_meta[0]['flip_direction'] @@ -251,10 +257,17 @@ def inference(self, img, img_meta, rescale): return output - def simple_test(self, img, img_meta, rescale=True): + def simple_test(self, img, img_meta, rescale=True, return_logit=False): """Simple test with single image.""" - seg_logit = self.inference(img, img_meta, rescale) - seg_pred = seg_logit.argmax(dim=1) + seg_logit = self.inference(img, img_meta, rescale, return_logit) + if return_logit: + seg_pred = seg_logit + else: + if seg_logit.shape[1] >= 2: + seg_pred = seg_logit.argmax(dim=1) + else: + seg_pred = seg_logit.squeeze(1) + seg_pred = (seg_pred > 0.5).int() if torch.onnx.is_in_onnx_export(): # our inference backend only support 4D output seg_pred = seg_pred.unsqueeze(0) @@ -264,7 +277,7 @@ def simple_test(self, img, img_meta, rescale=True): seg_pred = list(seg_pred) return seg_pred - def aug_test(self, imgs, img_metas, rescale=True): + def aug_test(self, imgs, img_metas, rescale=True, return_logit=False): """Test with augmentations. Only rescale=True is supported. @@ -272,12 +285,17 @@ def aug_test(self, imgs, img_metas, rescale=True): # aug_test rescale all imgs back to ori_shape for now assert rescale # to save memory, we get augmented seg logit inplace - seg_logit = self.inference(imgs[0], img_metas[0], rescale) + seg_logit = self.inference(imgs[0], img_metas[0], rescale, + return_logit) for i in range(1, len(imgs)): - cur_seg_logit = self.inference(imgs[i], img_metas[i], rescale) + cur_seg_logit = self.inference(imgs[i], img_metas[i], rescale, + return_logit) seg_logit += cur_seg_logit seg_logit /= len(imgs) - seg_pred = seg_logit.argmax(dim=1) + if return_logit: + seg_pred = seg_logit.squeeze(1) + else: + seg_pred = seg_logit.argmax(dim=1) seg_pred = seg_pred.cpu().numpy() # unravel batch dim seg_pred = list(seg_pred) diff --git a/setup.cfg b/setup.cfg index 8605ae9393..17ae8bfc9d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,6 +8,6 @@ line_length = 79 multi_line_output = 0 known_standard_library = setuptools known_first_party = mmseg -known_third_party = PIL,cityscapesscripts,cv2,detail,matplotlib,mmcv,numpy,onnxruntime,packaging,prettytable,pytest,pytorch_sphinx_theme,requests,scipy,seaborn,torch,ts +known_third_party = PIL,cityscapesscripts,cv2,detail,matplotlib,mmcv,numpy,onnxruntime,packaging,prettytable,pytest,pytorch_sphinx_theme,rarfile,requests,scipy,seaborn,torch,ts no_lines_before = STDLIB,LOCALFOLDER default_section = THIRDPARTY diff --git a/tools/convert_datasets/dut_omron.py b/tools/convert_datasets/dut_omron.py new file mode 100644 index 0000000000..bf331a50b9 --- /dev/null +++ b/tools/convert_datasets/dut_omron.py @@ -0,0 +1,71 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import os +import os.path as osp +import shutil +import tempfile +import zipfile + +import mmcv + +DUT_OMRON_LEN = 5168 + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Convert DUT-OMRON dataset to mmsegmentation format') + parser.add_argument('image_path', help='the path of DUT-OMRON-image.zip') + parser.add_argument( + 'mask_path', help='the path of DUT-OMRON-gt-pixelwise.zip.zip') + parser.add_argument('--tmp_dir', help='path of the temporary directory') + parser.add_argument('-o', '--out_dir', help='output path') + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + if args.out_dir is None: + out_dir = osp.join('data', 'DUT-OMRON') + else: + out_dir = args.out_dir + + print('Making directories...') + mmcv.mkdir_or_exist(out_dir) + mmcv.mkdir_or_exist(osp.join(out_dir, 'images')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'images', 'validation')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'annotations')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'annotations', 'validation')) + + print('Generating images...') + with tempfile.TemporaryDirectory(dir=args.tmp_dir) as tmp_dir: + zip_image = zipfile.ZipFile(args.image_path) + zip_image.extractall(tmp_dir) + zip_mask = zipfile.ZipFile(args.mask_path) + zip_mask.extractall(tmp_dir) + + image_dir = osp.join(tmp_dir, 'DUT-OMRON-image') + mask_dir = osp.join(tmp_dir, 'pixelwiseGT-new-PNG') + + assert len(os.listdir(image_dir)) == DUT_OMRON_LEN \ + and len(os.listdir(mask_dir)) == \ + DUT_OMRON_LEN, 'len(DUT-OMRON) != {}'.format(DUT_OMRON_LEN) + + for filename in sorted(os.listdir(image_dir)): + shutil.copy( + osp.join(image_dir, filename), + osp.join(out_dir, 'images', 'validation', + osp.splitext(filename)[0] + '.png')) + + for filename in sorted(os.listdir(mask_dir)): + img = mmcv.imread(osp.join(mask_dir, filename)) + mmcv.imwrite( + img[:, :, 0] // 128, + osp.join(out_dir, 'annotations', 'validation', + osp.splitext(filename)[0] + '.png')) + + print('Done!') + + +if __name__ == '__main__': + main() diff --git a/tools/convert_datasets/duts.py b/tools/convert_datasets/duts.py new file mode 100644 index 0000000000..dc370f9c8d --- /dev/null +++ b/tools/convert_datasets/duts.py @@ -0,0 +1,95 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import os +import os.path as osp +import shutil +import tempfile +import zipfile + +import mmcv + +TRAIN_LEN = 10553 +TEST_LEN = 5019 + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Convert DUTS dataset to mmsegmentation format') + parser.add_argument('trainset_path', help='the path of DUTS-TR.zip') + parser.add_argument('testset_path', help='the path of DUTS-TE.zip') + parser.add_argument('--tmp_dir', help='path of the temporary directory') + parser.add_argument('-o', '--out_dir', help='output path') + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + if args.out_dir is None: + out_dir = osp.join('data', 'DUTS') + else: + out_dir = args.out_dir + + print('Making directories...') + mmcv.mkdir_or_exist(out_dir) + mmcv.mkdir_or_exist(osp.join(out_dir, 'images')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'images', 'training')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'images', 'validation')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'annotations')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'annotations', 'training')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'annotations', 'validation')) + + print('Generating images...') + # DUTS-TR + with tempfile.TemporaryDirectory(dir=args.tmp_dir) as tmp_dir: + zip_file = zipfile.ZipFile(args.trainset_path) + zip_file.extractall(tmp_dir) + image_dir = osp.join(tmp_dir, 'DUTS-TR', 'DUTS-TR-Image') + mask_dir = osp.join(tmp_dir, 'DUTS-TR', 'DUTS-TR-Mask') + + assert len(os.listdir(image_dir)) == TRAIN_LEN \ + and len(os.listdir(mask_dir)) == \ + TRAIN_LEN, 'len(train_set) != {}'.format(TRAIN_LEN) + + for filename in sorted(os.listdir(image_dir)): + shutil.copy( + osp.join(image_dir, filename), + osp.join(out_dir, 'images', 'training', + osp.splitext(filename)[0] + '.png')) + + for filename in sorted(os.listdir(mask_dir)): + img = mmcv.imread(osp.join(mask_dir, filename)) + mmcv.imwrite( + img[:, :, 0] // 128, + osp.join(out_dir, 'annotations', 'training', + osp.splitext(filename)[0] + '.png')) + + # DUTS-TE + with tempfile.TemporaryDirectory(dir=args.tmp_dir) as tmp_dir: + zip_file = zipfile.ZipFile(args.testset_path) + zip_file.extractall(tmp_dir) + image_dir = osp.join(tmp_dir, 'DUTS-TE', 'DUTS-TE-Image') + mask_dir = osp.join(tmp_dir, 'DUTS-TE', 'DUTS-TE-Mask') + + assert len(os.listdir(image_dir)) == TEST_LEN \ + and len(os.listdir(mask_dir)) == \ + TEST_LEN, 'len(test_set) != {}'.format(TEST_LEN) + + for filename in sorted(os.listdir(image_dir)): + shutil.copy( + osp.join(image_dir, filename), + osp.join(out_dir, 'images', 'validation', + osp.splitext(filename)[0] + '.png')) + + for filename in sorted(os.listdir(mask_dir)): + img = mmcv.imread(osp.join(mask_dir, filename)) + mmcv.imwrite( + img[:, :, 0] // 128, + osp.join(out_dir, 'annotations', 'validation', + osp.splitext(filename)[0] + '.png')) + + print('Done!') + + +if __name__ == '__main__': + main() diff --git a/tools/convert_datasets/ecssd.py b/tools/convert_datasets/ecssd.py new file mode 100644 index 0000000000..ea557fb865 --- /dev/null +++ b/tools/convert_datasets/ecssd.py @@ -0,0 +1,70 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import os +import os.path as osp +import shutil +import tempfile +import zipfile + +import mmcv + +ECSSD_LEN = 1000 + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Convert ECSSD dataset to mmsegmentation format') + parser.add_argument('image_path', help='the path of images.zip') + parser.add_argument('mask_path', help='the path of ground_truth_mask.zip') + parser.add_argument('--tmp_dir', help='path of the temporary directory') + parser.add_argument('-o', '--out_dir', help='output path') + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + if args.out_dir is None: + out_dir = osp.join('data', 'ECSSD') + else: + out_dir = args.out_dir + + print('Making directories...') + mmcv.mkdir_or_exist(out_dir) + mmcv.mkdir_or_exist(osp.join(out_dir, 'images')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'images', 'validation')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'annotations')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'annotations', 'validation')) + + print('Generating images...') + with tempfile.TemporaryDirectory(dir=args.tmp_dir) as tmp_dir: + zip_image = zipfile.ZipFile(args.image_path) + zip_image.extractall(tmp_dir) + zip_mask = zipfile.ZipFile(args.mask_path) + zip_mask.extractall(tmp_dir) + + image_dir = osp.join(tmp_dir, 'images') + mask_dir = osp.join(tmp_dir, 'ground_truth_mask') + + assert len(os.listdir(image_dir)) == ECSSD_LEN \ + and len(os.listdir(mask_dir)) == \ + ECSSD_LEN, 'len(ECSSD) != {}'.format(ECSSD_LEN) + + for filename in sorted(os.listdir(image_dir)): + shutil.copy( + osp.join(image_dir, filename), + osp.join(out_dir, 'images', 'validation', + osp.splitext(filename)[0] + '.png')) + + for filename in sorted(os.listdir(mask_dir)): + img = mmcv.imread(osp.join(mask_dir, filename)) + mmcv.imwrite( + img[:, :, 0] // 128, + osp.join(out_dir, 'annotations', 'validation', + osp.splitext(filename)[0] + '.png')) + + print('Done!') + + +if __name__ == '__main__': + main() diff --git a/tools/convert_datasets/hku_is.py b/tools/convert_datasets/hku_is.py new file mode 100644 index 0000000000..46c0fd98e3 --- /dev/null +++ b/tools/convert_datasets/hku_is.py @@ -0,0 +1,67 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import os +import os.path as osp +import shutil +import tempfile + +import mmcv +import rarfile + +HKU_IS_LEN = 4447 + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Convert HKU-IS dataset to mmsegmentation format') + parser.add_argument('hkuis_path', help='the path of HKU-IS.rar') + parser.add_argument('--tmp_dir', help='path of the temporary directory') + parser.add_argument('-o', '--out_dir', help='output path') + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + if args.out_dir is None: + out_dir = osp.join('data', 'HKU-IS') + else: + out_dir = args.out_dir + + print('Making directories...') + mmcv.mkdir_or_exist(out_dir) + mmcv.mkdir_or_exist(osp.join(out_dir, 'images')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'images', 'validation')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'annotations')) + mmcv.mkdir_or_exist(osp.join(out_dir, 'annotations', 'validation')) + + print('Generating images...') + with tempfile.TemporaryDirectory(dir=args.tmp_dir) as tmp_dir: + rar_file = rarfile.RarFile(args.hkuis_path) + rar_file.extractall(tmp_dir) + + image_dir = osp.join(tmp_dir, 'HKU-IS', 'imgs') + mask_dir = osp.join(tmp_dir, 'HKU-IS', 'gt') + + assert len(os.listdir(image_dir)) == HKU_IS_LEN \ + and len(os.listdir(mask_dir)) == \ + HKU_IS_LEN, 'len(HKU-IS) != {}'.format(HKU_IS_LEN) + + for filename in sorted(os.listdir(image_dir)): + shutil.copy( + osp.join(image_dir, filename), + osp.join(out_dir, 'images', 'validation', + osp.splitext(filename)[0] + '.png')) + + for filename in sorted(os.listdir(mask_dir)): + img = mmcv.imread(osp.join(mask_dir, filename)) + mmcv.imwrite( + img[:, :, 0] // 128, + osp.join(out_dir, 'annotations', 'validation', + osp.splitext(filename)[0] + '.png')) + + print('Done!') + + +if __name__ == '__main__': + main()