From bca62ca9570232941ce1d3e23f341ef3fab56b5c Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sat, 19 Jun 2021 11:11:34 +0300 Subject: [PATCH 1/3] Add import order check in pylint --- .pylintrc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.pylintrc b/.pylintrc index 09864236e6..2c9edce81b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -61,11 +61,11 @@ enable= E0001,E0100,E0101,E0102,E0103,E0104,E0105,E0106,E0107,E0110, E0702,E0703,E0704,E0710,E0711,E0712,E1003,E1102,E1111,E0112, E1120,E1121,E1123,E1124,E1125,E1126,E1127,E1132,E1200,E1201, E1205,E1206,E1300,E1301,E1302,E1303,E1304,E1305,E1306, - C0123,C0200,C0303,C1001, + C0123,C0200,C0303,C0411,C1001, W0101,W0102,W0104,W0105,W0106,W0107,W0108,W0109,W0110,W0120, W0122,W0124,W0150,W0199,W0221,W0222,W0233,W0404,W0410,W0601, W0602,W0604,W0611,W0612,W0622,W0623,W0702,W0705,W0711,W1300, - W1301,W1302,W1303,,W1305,W1306,W1307 + W1301,W1302,W1303,W1305,W1306,W1307 R0102,R0202,R0203 @@ -112,7 +112,9 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme [BASIC] # Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ +good-names=i,j,k,ex,Run,_,x,y,w,h,d,c,id,it + +allowed-redefined-builtins=id,format,dir # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata @@ -300,7 +302,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. -generated-members= +generated-members=on_error # List of decorators that produce context managers, such as # contextlib.contextmanager. Add to this list to register other decorators that From 011537258c493ea294186c349099c86a23cae7db Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sat, 19 Jun 2021 11:12:04 +0300 Subject: [PATCH 2/3] Fix some linter problems --- datumaro/components/dataset.py | 13 +++++++++---- datumaro/components/errors.py | 4 ++++ datumaro/components/extractor.py | 4 ++-- datumaro/plugins/voc_format/converter.py | 2 +- tests/test_voc_format.py | 4 ++-- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/datumaro/components/dataset.py b/datumaro/components/dataset.py index ea82cc6fc9..51f4e19f9d 100644 --- a/datumaro/components/dataset.py +++ b/datumaro/components/dataset.py @@ -18,7 +18,8 @@ IExtractor, LabelCategories, AnnotationType, DatasetItem, DEFAULT_SUBSET_NAME, Transform) from datumaro.components.environment import Environment -from datumaro.components.errors import DatumaroError, RepeatedItemError +from datumaro.components.errors import (CategoriesRedefinedError, + DatumaroError, RepeatedItemError) from datumaro.util import error_rollback from datumaro.util.log_utils import logging_disabled @@ -78,7 +79,7 @@ def subsets(self): class DatasetItemStorageDatasetView(IDataset): class Subset(IDataset): - def __init__(self, parent, name): + def __init__(self, parent: 'DatasetItemStorageDatasetView', name: str): super().__init__() self.parent = parent self.name = name @@ -319,6 +320,11 @@ def categories(self) -> CategoriesInfo: else: return self._source.categories() + def define_categories(self, categories: CategoriesInfo): + if self._categories or self._source is not None: + raise CategoriesRedefinedError() + self._categories = categories + def put(self, item): is_new = self._storage.put(item) @@ -459,8 +465,7 @@ def __init__(self, source: IDataset = None, self._source_path = None def define_categories(self, categories: Dict): - assert not self._data._categories and self._data._source is None - self._data._categories = categories + self._data.define_categories(categories) def init_cache(self): self._data.init_cache() diff --git a/datumaro/components/errors.py b/datumaro/components/errors.py index 717399b060..ac23e3a9b0 100644 --- a/datumaro/components/errors.py +++ b/datumaro/components/errors.py @@ -17,6 +17,10 @@ class RepeatedItemError(DatasetError): def __str__(self): return "Item %s is repeated in the source sequence." % (self.item_id, ) +class CategoriesRedefinedError(DatasetError): + def __str__(self): + return "Categories can only be set once for a dataset" + @attrs class MismatchingImageInfoError(DatasetError): a = attrib() diff --git a/datumaro/components/extractor.py b/datumaro/components/extractor.py index 91a5654f0b..0927795c7b 100644 --- a/datumaro/components/extractor.py +++ b/datumaro/components/extractor.py @@ -6,12 +6,12 @@ from enum import Enum, auto from glob import iglob from typing import Callable, Iterable, List, Dict, Optional -import numpy as np import os import os.path as osp -import attr from attr import attrs, attrib +import attr +import numpy as np from datumaro.util.image import Image from datumaro.util.attrs_util import not_empty, default_if_none diff --git a/datumaro/plugins/voc_format/converter.py b/datumaro/plugins/voc_format/converter.py index 96252d1d8c..43335a1236 100644 --- a/datumaro/plugins/voc_format/converter.py +++ b/datumaro/plugins/voc_format/converter.py @@ -26,7 +26,7 @@ ) -def _convert_attr(name, attributes, type_conv, default=None, warn=True): +def _convert_attr(name, attributes, type_conv, default=None): d = object() value = attributes.get(name, d) if value is d: diff --git a/tests/test_voc_format.py b/tests/test_voc_format.py index 328ed83dc6..fd0b22343f 100644 --- a/tests/test_voc_format.py +++ b/tests/test_voc_format.py @@ -1,10 +1,10 @@ from collections import OrderedDict from functools import partial -import numpy as np +from unittest import TestCase import os import os.path as osp -from unittest import TestCase +import numpy as np from datumaro.components.extractor import (Extractor, DatasetItem, AnnotationType, Label, Bbox, Mask, LabelCategories, From 7c6922dd8c4068dd782a43b89ca8fe367dc8ca96 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sun, 20 Jun 2021 14:48:00 +0300 Subject: [PATCH 3/3] Remove warning suppression comments --- datumaro/components/dataset.py | 2 -- datumaro/components/extractor.py | 6 +++--- datumaro/components/project.py | 11 ++++------- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/datumaro/components/dataset.py b/datumaro/components/dataset.py index 51f4e19f9d..3648e3918f 100644 --- a/datumaro/components/dataset.py +++ b/datumaro/components/dataset.py @@ -2,8 +2,6 @@ # # SPDX-License-Identifier: MIT -#pylint: disable=redefined-builtin - from contextlib import contextmanager from enum import Enum, auto from typing import Iterable, Iterator, Optional, Tuple, Union, Dict, List diff --git a/datumaro/components/extractor.py b/datumaro/components/extractor.py index 0927795c7b..19405eebe7 100644 --- a/datumaro/components/extractor.py +++ b/datumaro/components/extractor.py @@ -541,7 +541,7 @@ def wrap(item, **kwargs): CategoriesInfo = Dict[AnnotationType, Categories] -class IExtractor: #pylint: disable=redefined-builtin +class IExtractor: def __iter__(self) -> Iterable[DatasetItem]: raise NotImplementedError() @@ -614,7 +614,7 @@ def categories(_): def categories(self): return {} - def get(self, id, subset=None): #pylint: disable=redefined-builtin + def get(self, id, subset=None): subset = subset or DEFAULT_SUBSET_NAME for item in self: if item.id == id and item.subset == subset: @@ -638,7 +638,7 @@ def __iter__(self): def __len__(self): return len(self._items) - def get(self, id, subset=None): #pylint: disable=redefined-builtin + def get(self, id, subset=None): assert subset == self._subset, '%s != %s' % (subset, self._subset) return super().get(id, subset or self._subset) diff --git a/datumaro/components/project.py b/datumaro/components/project.py index 829fde46ba..95b10a94c3 100644 --- a/datumaro/components/project.py +++ b/datumaro/components/project.py @@ -38,7 +38,7 @@ def __len__(self): def categories(self): return self.parent.categories() - def get(self, id, subset=None): #pylint: disable=redefined-builtin + def get(self, id, subset=None): subset = subset or self.name assert subset == self.name, '%s != %s' % (subset, self.name) return super().get(id, subset) @@ -137,15 +137,13 @@ def categories(self): def __len__(self): return sum(len(s) for s in self._subsets.values()) - def get(self, id, subset=None, \ - path=None): #pylint: disable=redefined-builtin + def get(self, id, subset=None, path=None): if path: source = path[0] return self._sources[source].get(id=id, subset=subset) return self._subsets.get(subset, {}).get(id) - def put(self, item, id=None, subset=None, \ - path=None): #pylint: disable=redefined-builtin + def put(self, item, id=None, subset=None, path=None): if path is None: path = item.path @@ -277,8 +275,7 @@ def categories(_): return self.transform(_DatasetFilter) - def export(self, save_dir: str, format, \ - **kwargs): #pylint: disable=redefined-builtin + def export(self, save_dir: str, format, **kwargs): dataset = Dataset.from_extractors(self, env=self.env) dataset.export(save_dir, format, **kwargs)