From 8e297ffef23b060216f9c550488c03e008fe879d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Haian=20Huang=28=E6=B7=B1=E5=BA=A6=E7=9C=B8=29?= <1286304229@qq.com> Date: Tue, 2 Nov 2021 21:19:06 +0800 Subject: [PATCH 1/5] Fix aug test error when the number of prediction bboxes is 0 (#6398) * Fix aug test error when the number of prediction bboxes is 0 * test * test * fix lint --- mmdet/models/dense_heads/dense_test_mixins.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mmdet/models/dense_heads/dense_test_mixins.py b/mmdet/models/dense_heads/dense_test_mixins.py index 84bbe6f1109..3421548955d 100644 --- a/mmdet/models/dense_heads/dense_test_mixins.py +++ b/mmdet/models/dense_heads/dense_test_mixins.py @@ -93,7 +93,9 @@ def aug_test_bboxes(self, feats, img_metas, rescale=False): if merged_bboxes.numel() == 0: det_bboxes = torch.cat([merged_bboxes, merged_scores[:, None]], -1) - return det_bboxes, merged_labels + return [ + (det_bboxes, merged_labels), + ] det_bboxes, keep_idxs = batched_nms(merged_bboxes, merged_scores, merged_labels, self.test_cfg.nms) From 892f9108c9180a090d1034fbf57d6de91e2e0f58 Mon Sep 17 00:00:00 2001 From: "PJLAB\\huanghaian" <1286304229@qq.com> Date: Wed, 3 Nov 2021 14:14:56 +0800 Subject: [PATCH 2/5] Support custom pin_memory and persistent_workers --- mmdet/apis/train.py | 5 ++++- mmdet/datasets/builder.py | 18 ++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/mmdet/apis/train.py b/mmdet/apis/train.py index b8b6eefc8ab..50f02d37a5f 100644 --- a/mmdet/apis/train.py +++ b/mmdet/apis/train.py @@ -71,7 +71,10 @@ def train_detector(model, num_gpus=len(cfg.gpu_ids), dist=distributed, seed=cfg.seed, - runner_type=runner_type) for ds in dataset + runner_type=runner_type, + pin_memory=cfg.data.get('pin_memory', False), + persistent_workers=cfg.data.get('persistent_workers', False)) + for ds in dataset ] # put model on gpus diff --git a/mmdet/datasets/builder.py b/mmdet/datasets/builder.py index c0c118a0fed..755792f1942 100644 --- a/mmdet/datasets/builder.py +++ b/mmdet/datasets/builder.py @@ -7,7 +7,7 @@ import numpy as np from mmcv.parallel import collate from mmcv.runner import get_dist_info -from mmcv.utils import Registry, build_from_cfg +from mmcv.utils import TORCH_VERSION, Registry, build_from_cfg, digit_version from torch.utils.data import DataLoader from .samplers import (DistributedGroupSampler, DistributedSampler, @@ -90,6 +90,8 @@ def build_dataloader(dataset, shuffle=True, seed=None, runner_type='EpochBasedRunner', + pin_memory=False, + persistent_workers=False, **kwargs): """Build PyTorch DataLoader. @@ -106,7 +108,15 @@ def build_dataloader(dataset, dist (bool): Distributed training/test or not. Default: True. shuffle (bool): Whether to shuffle the data at every epoch. Default: True. + seed (int, Optional): Seed to be used. Default: None. runner_type (str): Type of runner. Default: `EpochBasedRunner` + pin_memory (bool): Whether to use pin_memory in DataLoader. + Default: False. + persistent_workers (bool): If True, the data loader will not shutdown + the worker processes after a dataset has been consumed once. + This allows to maintain the workers Dataset instances alive. + The argument also has effect in PyTorch>=1.7.0. + Default: False. kwargs: any keyword argument to be used to initialize DataLoader Returns: @@ -163,6 +173,10 @@ def build_dataloader(dataset, worker_init_fn, num_workers=num_workers, rank=rank, seed=seed) if seed is not None else None + if (TORCH_VERSION != 'parrots' + and digit_version(TORCH_VERSION) >= digit_version('1.7.0')): + kwargs['persistent_workers'] = persistent_workers + data_loader = DataLoader( dataset, batch_size=batch_size, @@ -170,7 +184,7 @@ def build_dataloader(dataset, num_workers=num_workers, batch_sampler=batch_sampler, collate_fn=partial(collate, samples_per_gpu=samples_per_gpu), - pin_memory=False, + pin_memory=pin_memory, worker_init_fn=init_fn, **kwargs) From 4db90f37a6f023fb2c1b3480f72bb73e160cd8ed Mon Sep 17 00:00:00 2001 From: "PJLAB\\huanghaian" <1286304229@qq.com> Date: Mon, 8 Nov 2021 09:11:28 +0800 Subject: [PATCH 3/5] fix comment --- mmdet/datasets/builder.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mmdet/datasets/builder.py b/mmdet/datasets/builder.py index 755792f1942..88413690768 100644 --- a/mmdet/datasets/builder.py +++ b/mmdet/datasets/builder.py @@ -2,6 +2,7 @@ import copy import platform import random +import warnings from functools import partial import numpy as np @@ -176,6 +177,9 @@ def build_dataloader(dataset, if (TORCH_VERSION != 'parrots' and digit_version(TORCH_VERSION) >= digit_version('1.7.0')): kwargs['persistent_workers'] = persistent_workers + elif persistent_workers is True: + warnings.warn('persistent_workers is invalid because your pytorch ' + 'version is lower than 1.7.0') data_loader = DataLoader( dataset, From 6082efe523bd5ffcdd1d60ac43952887b9af8fa1 Mon Sep 17 00:00:00 2001 From: "PJLAB\\huanghaian" <1286304229@qq.com> Date: Mon, 8 Nov 2021 11:49:40 +0800 Subject: [PATCH 4/5] fix docstr --- mmdet/datasets/builder.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mmdet/datasets/builder.py b/mmdet/datasets/builder.py index 88413690768..61626394ac0 100644 --- a/mmdet/datasets/builder.py +++ b/mmdet/datasets/builder.py @@ -115,9 +115,8 @@ def build_dataloader(dataset, Default: False. persistent_workers (bool): If True, the data loader will not shutdown the worker processes after a dataset has been consumed once. - This allows to maintain the workers Dataset instances alive. - The argument also has effect in PyTorch>=1.7.0. - Default: False. + This allows to maintain the workers `Dataset` instances alive. + This argument is only valid when PyTorch>=1.7.0. Default: False. kwargs: any keyword argument to be used to initialize DataLoader Returns: From 56943929092748301a853df4030a326070008fb1 Mon Sep 17 00:00:00 2001 From: "PJLAB\\huanghaian" <1286304229@qq.com> Date: Tue, 9 Nov 2021 13:52:21 +0800 Subject: [PATCH 5/5] remove pin_memory --- mmdet/apis/train.py | 1 - mmdet/datasets/builder.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/mmdet/apis/train.py b/mmdet/apis/train.py index 50f02d37a5f..d3b2fc3cab6 100644 --- a/mmdet/apis/train.py +++ b/mmdet/apis/train.py @@ -72,7 +72,6 @@ def train_detector(model, dist=distributed, seed=cfg.seed, runner_type=runner_type, - pin_memory=cfg.data.get('pin_memory', False), persistent_workers=cfg.data.get('persistent_workers', False)) for ds in dataset ] diff --git a/mmdet/datasets/builder.py b/mmdet/datasets/builder.py index 61626394ac0..ab0d3a801b5 100644 --- a/mmdet/datasets/builder.py +++ b/mmdet/datasets/builder.py @@ -91,7 +91,6 @@ def build_dataloader(dataset, shuffle=True, seed=None, runner_type='EpochBasedRunner', - pin_memory=False, persistent_workers=False, **kwargs): """Build PyTorch DataLoader. @@ -111,8 +110,6 @@ def build_dataloader(dataset, Default: True. seed (int, Optional): Seed to be used. Default: None. runner_type (str): Type of runner. Default: `EpochBasedRunner` - pin_memory (bool): Whether to use pin_memory in DataLoader. - Default: False. persistent_workers (bool): If True, the data loader will not shutdown the worker processes after a dataset has been consumed once. This allows to maintain the workers `Dataset` instances alive. @@ -187,7 +184,7 @@ def build_dataloader(dataset, num_workers=num_workers, batch_sampler=batch_sampler, collate_fn=partial(collate, samples_per_gpu=samples_per_gpu), - pin_memory=pin_memory, + pin_memory=False, worker_init_fn=init_fn, **kwargs)