From 1f9dd646d4383fee6cfd2fbdf32f3eca92245fca Mon Sep 17 00:00:00 2001 From: Jan Ittner Date: Thu, 1 Oct 2020 08:49:25 +0200 Subject: [PATCH 1/3] replace wildcard imports with explicit import statements --- src/sklearndf/_sklearndf.py | 12 +++--------- src/sklearndf/_wrapper.py | 18 ++++++++++++++++-- .../classification/_classification.py | 2 +- src/sklearndf/pipeline/_learner_pipeline.py | 2 +- src/sklearndf/pipeline/_pipeline.py | 2 +- src/sklearndf/regression/_regression.py | 3 ++- src/sklearndf/regression/_regression_v0_22.py | 7 ++----- src/sklearndf/regression/_regression_v0_23.py | 2 +- .../transformation/_transformation.py | 6 +++--- src/sklearndf/transformation/_wrapper.py | 2 +- src/sklearndf/transformation/extra/_extra.py | 7 ++----- test/test/sklearndf/__init__.py | 3 +-- .../sklearndf/pipeline/test_pipeline_df.py | 2 +- test/test/sklearndf/test_classification.py | 3 +-- test/test/sklearndf/test_regression.py | 3 +-- test/test/sklearndf/test_sklearn_coverage.py | 9 ++------- .../sklearndf/transformation/test_imputers.py | 3 +-- .../transformation/test_transformation.py | 3 ++- 18 files changed, 42 insertions(+), 47 deletions(-) diff --git a/src/sklearndf/_sklearndf.py b/src/sklearndf/_sklearndf.py index d860c0b5..dc98ca12 100644 --- a/src/sklearndf/_sklearndf.py +++ b/src/sklearndf/_sklearndf.py @@ -4,7 +4,7 @@ import logging from abc import ABCMeta, abstractmethod -from typing import * +from typing import Any, List, Mapping, Optional, Sequence, Type, TypeVar, Union, cast import pandas as pd from sklearn.base import ( @@ -19,13 +19,7 @@ log = logging.getLogger(__name__) -__all__ = [ - "EstimatorDF", - "LearnerDF", - "ClassifierDF", - "RegressorDF", - "TransformerDF", -] +__all__ = ["EstimatorDF", "LearnerDF", "ClassifierDF", "RegressorDF", "TransformerDF"] # # type variables @@ -401,4 +395,4 @@ def classes_(self) -> Sequence[Any]: :return: the classes predicted by this classifier """ - pass \ No newline at end of file + pass diff --git a/src/sklearndf/_wrapper.py b/src/sklearndf/_wrapper.py index 6ccbcf67..c2b56348 100644 --- a/src/sklearndf/_wrapper.py +++ b/src/sklearndf/_wrapper.py @@ -16,7 +16,21 @@ import re from abc import ABCMeta, abstractmethod from functools import update_wrapper -from typing import * +from typing import ( + Optional, + Iterable, + Generic, + Mapping, + Sequence, + Any, + Union, + Callable, + TypeVar, + Type, + AnyStr, + Dict, + List, +) import numpy as np import pandas as pd @@ -29,7 +43,7 @@ ) from pytools.api import inheritdoc -from sklearndf import (ClassifierDF, EstimatorDF, LearnerDF, RegressorDF, TransformerDF) +from sklearndf import ClassifierDF, EstimatorDF, LearnerDF, RegressorDF, TransformerDF log = logging.getLogger(__name__) diff --git a/src/sklearndf/classification/_classification.py b/src/sklearndf/classification/_classification.py index 30a433ee..19780e40 100644 --- a/src/sklearndf/classification/_classification.py +++ b/src/sklearndf/classification/_classification.py @@ -3,7 +3,7 @@ """ import logging from abc import ABCMeta -from typing import * +from typing import Any, List, Optional, Sequence, Union import numpy as np import pandas as pd diff --git a/src/sklearndf/pipeline/_learner_pipeline.py b/src/sklearndf/pipeline/_learner_pipeline.py index ff1dc30f..1e0dff77 100644 --- a/src/sklearndf/pipeline/_learner_pipeline.py +++ b/src/sklearndf/pipeline/_learner_pipeline.py @@ -4,7 +4,7 @@ import logging from abc import ABCMeta, abstractmethod -from typing import * +from typing import Any, Generic, List, Optional, Sequence, TypeVar, Union import pandas as pd from sklearn.base import BaseEstimator diff --git a/src/sklearndf/pipeline/_pipeline.py b/src/sklearndf/pipeline/_pipeline.py index 3fc19073..fe115942 100644 --- a/src/sklearndf/pipeline/_pipeline.py +++ b/src/sklearndf/pipeline/_pipeline.py @@ -4,7 +4,7 @@ import logging from abc import ABCMeta -from typing import * +from typing import Iterator, List, Sequence, Tuple, Union, cast import numpy as np import pandas as pd diff --git a/src/sklearndf/regression/_regression.py b/src/sklearndf/regression/_regression.py index d81d932c..a201e199 100644 --- a/src/sklearndf/regression/_regression.py +++ b/src/sklearndf/regression/_regression.py @@ -3,7 +3,7 @@ """ import logging from abc import ABCMeta -from typing import * +from typing import Any, Generic, Optional, TypeVar, Union import pandas as pd from sklearn.base import RegressorMixin @@ -56,6 +56,7 @@ from .. import RegressorDF, TransformerDF from .._wrapper import _MetaRegressorWrapperDF, _RegressorWrapperDF, df_estimator + # noinspection PyProtectedMember from ..transformation._wrapper import _ColumnPreservingTransformerWrapperDF diff --git a/src/sklearndf/regression/_regression_v0_22.py b/src/sklearndf/regression/_regression_v0_22.py index 0f331ce8..6297d3e9 100644 --- a/src/sklearndf/regression/_regression_v0_22.py +++ b/src/sklearndf/regression/_regression_v0_22.py @@ -3,16 +3,13 @@ from sklearn 0.22 onwards """ import logging -from typing import * +from typing import TypeVar from sklearn.base import RegressorMixin from sklearn.ensemble import StackingRegressor from .. import RegressorDF -from .._wrapper import ( - _StackingRegressorWrapperDF, - df_estimator, -) +from .._wrapper import _StackingRegressorWrapperDF, df_estimator # noinspection PyProtectedMember diff --git a/src/sklearndf/regression/_regression_v0_23.py b/src/sklearndf/regression/_regression_v0_23.py index e4d19d16..daa833f8 100644 --- a/src/sklearndf/regression/_regression_v0_23.py +++ b/src/sklearndf/regression/_regression_v0_23.py @@ -4,7 +4,7 @@ """ import logging -from typing import * +from typing import TypeVar from sklearn.base import RegressorMixin from sklearn.linear_model import GammaRegressor, PoissonRegressor, TweedieRegressor diff --git a/src/sklearndf/transformation/_transformation.py b/src/sklearndf/transformation/_transformation.py index 9b70f55d..ffa2df90 100644 --- a/src/sklearndf/transformation/_transformation.py +++ b/src/sklearndf/transformation/_transformation.py @@ -19,8 +19,7 @@ import logging from abc import ABCMeta from functools import reduce -from typing import * -from typing import List +from typing import Iterable, List, TypeVar import numpy as np import pandas as pd @@ -228,7 +227,8 @@ def _validate_delegate_estimator(self) -> None: if non_compliant_transformers: raise ValueError( f"{ColumnTransformerDF.__name__} only accepts strings or " - f"instances of " f"{TransformerDF.__name__} as valid transformers, but " + f"instances of " + f"{TransformerDF.__name__} as valid transformers, but " f'also got: {", ".join(non_compliant_transformers)}' ) diff --git a/src/sklearndf/transformation/_wrapper.py b/src/sklearndf/transformation/_wrapper.py index 76bc86b7..0f301e66 100644 --- a/src/sklearndf/transformation/_wrapper.py +++ b/src/sklearndf/transformation/_wrapper.py @@ -4,7 +4,7 @@ import logging from abc import ABCMeta, abstractmethod -from typing import * +from typing import Any, Generic, Optional, TypeVar, Union import pandas as pd from sklearn.base import TransformerMixin diff --git a/src/sklearndf/transformation/extra/_extra.py b/src/sklearndf/transformation/extra/_extra.py index 8250becb..253f8188 100644 --- a/src/sklearndf/transformation/extra/_extra.py +++ b/src/sklearndf/transformation/extra/_extra.py @@ -4,16 +4,13 @@ import logging from abc import ABCMeta -from typing import * +from typing import Any, Mapping, Optional, TypeVar, Union import pandas as pd from boruta import BorutaPy from sklearn.base import BaseEstimator -from .._wrapper import ( - _ColumnSubsetTransformerWrapperDF, - _NDArrayTransformerWrapperDF, -) +from .._wrapper import _ColumnSubsetTransformerWrapperDF, _NDArrayTransformerWrapperDF from ... import TransformerDF from ..._wrapper import _MetaEstimatorWrapperDF, df_estimator diff --git a/test/test/sklearndf/__init__.py b/test/test/sklearndf/__init__.py index 4cbc367b..fe8ec62c 100644 --- a/test/test/sklearndf/__init__.py +++ b/test/test/sklearndf/__init__.py @@ -1,8 +1,7 @@ import re import sys from distutils import version -from typing import * -from typing import Type +from typing import Any, Dict, Iterable, List, Optional, Set, Type, Union import pandas as pd import sklearn diff --git a/test/test/sklearndf/pipeline/test_pipeline_df.py b/test/test/sklearndf/pipeline/test_pipeline_df.py index 5fb48adc..55d98dc8 100644 --- a/test/test/sklearndf/pipeline/test_pipeline_df.py +++ b/test/test/sklearndf/pipeline/test_pipeline_df.py @@ -6,7 +6,7 @@ import time from distutils.version import LooseVersion from tempfile import mkdtemp -from typing import * +from typing import Any, Dict, Mapping import joblib import numpy as np diff --git a/test/test/sklearndf/test_classification.py b/test/test/sklearndf/test_classification.py index 15c35f50..f7c44f42 100644 --- a/test/test/sklearndf/test_classification.py +++ b/test/test/sklearndf/test_classification.py @@ -1,9 +1,8 @@ from itertools import chain -from typing import * +from typing import Type import numpy as np import pandas as pd -# noinspection PyPackageRequirements import pytest from sklearn.multioutput import ClassifierChain, MultiOutputClassifier diff --git a/test/test/sklearndf/test_regression.py b/test/test/sklearndf/test_regression.py index 9c057f45..55187eec 100644 --- a/test/test/sklearndf/test_regression.py +++ b/test/test/sklearndf/test_regression.py @@ -1,7 +1,6 @@ -from typing import * +from typing import List, Type import pandas as pd -# noinspection PyPackageRequirements import pytest from sklearn.multioutput import MultiOutputRegressor, RegressorChain diff --git a/test/test/sklearndf/test_sklearn_coverage.py b/test/test/sklearndf/test_sklearn_coverage.py index ff5e9577..3cfaab66 100644 --- a/test/test/sklearndf/test_sklearn_coverage.py +++ b/test/test/sklearndf/test_sklearn_coverage.py @@ -1,6 +1,5 @@ import itertools -from typing import * -from typing import Dict +from typing import Dict, Iterable, List, Optional, Type, Union import pytest import sklearn @@ -19,11 +18,7 @@ from sklearndf import EstimatorDF from test import check_sklearn_version from test.conftest import UNSUPPORTED_SKLEARN_PACKAGES -from test.sklearndf import ( - find_all_submodules, - list_classes, - sklearn_delegate_classes, -) +from test.sklearndf import find_all_submodules, list_classes, sklearn_delegate_classes Module = type(sklearn) diff --git a/test/test/sklearndf/transformation/test_imputers.py b/test/test/sklearndf/transformation/test_imputers.py index 2cdaa501..61d11b71 100644 --- a/test/test/sklearndf/transformation/test_imputers.py +++ b/test/test/sklearndf/transformation/test_imputers.py @@ -1,10 +1,9 @@ import itertools import logging -from typing import * +from typing import Type import numpy as np import pandas as pd -# noinspection PyPackageRequirements import pytest import sklearndf.transformation diff --git a/test/test/sklearndf/transformation/test_transformation.py b/test/test/sklearndf/transformation/test_transformation.py index 13ac245b..ab9484cf 100644 --- a/test/test/sklearndf/transformation/test_transformation.py +++ b/test/test/sklearndf/transformation/test_transformation.py @@ -1,7 +1,8 @@ -from typing import * +from typing import Type import numpy as np import pandas as pd + # noinspection PyPackageRequirements import pytest import sklearn From 46bf39c6dd6200544616bf13ad67eb4bd5b726c1 Mon Sep 17 00:00:00 2001 From: Jan Ittner Date: Thu, 1 Oct 2020 09:34:20 +0200 Subject: [PATCH 2/3] upgrade black to v20.8b1 --- .pre-commit-config.yaml | 7 ++++--- environment.yml | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d5c980d..3cb791c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,6 @@ repos: - - repo: https://github.com/ambv/black - rev: stable + - repo: https://github.com/psf/black + rev: 20.8b1 hooks: - - id: black + - id: black + language_version: python3 diff --git a/environment.yml b/environment.yml index 09661bef..5fecca68 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - astropy - conda-forge dependencies: - - black = 19.* + - black = 20.8b1 - boruta_py = 0.3.* - conda-build - conda-verify From 9598b101b4cbf7523baa61c2e8ba6ace81e14cf4 Mon Sep 17 00:00:00 2001 From: Jan Ittner Date: Thu, 1 Oct 2020 11:05:51 +0200 Subject: [PATCH 3/3] upgrade black to v20.8b1 --- test/test/sklearndf/__init__.py | 25 ++++++++++--------- test/test/sklearndf/test_regression.py | 5 ++-- .../transformation/test_transformation.py | 16 ++++++------ 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/test/test/sklearndf/__init__.py b/test/test/sklearndf/__init__.py index fe8ec62c..1f7182e6 100644 --- a/test/test/sklearndf/__init__.py +++ b/test/test/sklearndf/__init__.py @@ -1,7 +1,7 @@ import re import sys from distutils import version -from typing import Any, Dict, Iterable, List, Optional, Set, Type, Union +from typing import Any, Iterable, List, Mapping, Optional, Set, Type, Union import pandas as pd import sklearn @@ -13,13 +13,14 @@ Module: type = Any -def find_all_classes(*modules: Module) -> Set[type]: +def find_all_classes(*modules: Module) -> Set[Type[_EstimatorWrapperDF]]: """ Finds all Class members in given module/modules. """ - types: Set[type] = set() + types: Set[Type[_EstimatorWrapperDF]] = set() def _add_classes_from_module(_m: Module) -> None: for member in vars(module).values(): if isinstance(member, type): + member: Type[_EstimatorWrapperDF] types.add(member) for module in modules: @@ -39,8 +40,8 @@ def find_all_submodules(parent_module: Module) -> Set[Module]: def sklearn_delegate_classes( - module: Module -) -> Dict[BaseEstimator, _EstimatorWrapperDF]: + module: Module, +) -> Mapping[Type[BaseEstimator], Type[_EstimatorWrapperDF]]: """ Creates a dictionary mapping from sklearndf -> sklearn classes. """ return { df_class.__wrapped__: df_class @@ -53,7 +54,7 @@ def list_classes( from_modules: Union[Module, Iterable[Module]], matching: str, excluding: Optional[Union[str, Iterable[str]]] = None, -) -> List[Type]: +) -> List[Type[_EstimatorWrapperDF]]: """ Helper to return all classes with matching name from Python module(s) """ if not isinstance(from_modules, Iterable): @@ -72,7 +73,7 @@ def list_classes( def get_sklearndf_wrapper_class( to_wrap: Type[BaseEstimator], from_module=None -) -> _EstimatorWrapperDF: +) -> Type[_EstimatorWrapperDF]: """ Helper to return the wrapped counterpart for a sklearn class """ try: return sklearn_delegate_classes(from_module)[to_wrap] @@ -91,10 +92,10 @@ def check_expected_not_fitted_error(estimator: Union[LearnerDF, TransformerDF]): test_x = pd.DataFrame(data=list(range(10))) def check_sklearndf_call( - func_to_call: str, estimator: Union[LearnerDF, TransformerDF] + func_to_call: str, _estimator: Union[LearnerDF, TransformerDF] ) -> None: try: - getattr(estimator, func_to_call)(X=test_x) + getattr(_estimator, func_to_call)(X=test_x) except sklearn.exceptions.NotFittedError: # This is the expected error, that sklearn[df] should raise return @@ -107,16 +108,16 @@ def check_sklearndf_call( else: x = test_x.values.reshape(-1) - getattr(estimator.native_estimator, func_to_call)(x) + getattr(_estimator.native_estimator, func_to_call)(x) except sklearn.exceptions.NotFittedError: raise AssertionError( "sklearndf did not return an expected NotFittedError" - f" for {estimator.__class__.__name__}" + f" for {_estimator.__class__.__name__}" ) except Exception as sklearn_exception: assert repr(sklearndf_exception) == repr(sklearn_exception), ( "sklearndf raised a different error as sklearn" - f" for {estimator.__class__.__name__}:" + f" for {_estimator.__class__.__name__}:" f"sklearndf: {repr(sklearndf_exception)} \n" f"sklearn: {repr(sklearn_exception)}" ) diff --git a/test/test/sklearndf/test_regression.py b/test/test/sklearndf/test_regression.py index 55187eec..fc4b439c 100644 --- a/test/test/sklearndf/test_regression.py +++ b/test/test/sklearndf/test_regression.py @@ -6,6 +6,7 @@ import sklearndf.regression from sklearndf import RegressorDF, TransformerDF +from sklearndf._wrapper import _EstimatorWrapperDF from sklearndf.regression import ( IsotonicRegressionDF, LinearRegressionDF, @@ -14,7 +15,7 @@ ) from test.sklearndf import check_expected_not_fitted_error, list_classes -REGRESSORS_TO_TEST: List[Type] = list_classes( +REGRESSORS_TO_TEST: List[Type[_EstimatorWrapperDF]] = list_classes( from_modules=sklearndf.regression, matching=r".*DF", excluding=[RegressorDF.__name__, TransformerDF.__name__, r".*WrapperDF"], @@ -24,7 +25,7 @@ "MultiOutputRegressorDF": {"estimator": RandomForestRegressorDF()}, "RegressorChainDF": {"base_estimator": RandomForestRegressorDF()}, "VotingRegressorDF": { - "estimators": [("rfr", RandomForestRegressorDF()), ("svmr", SVRDF())] + "estimators": [("rfr", RandomForestRegressorDF()), ("svr", SVRDF())] }, "StackingRegressorDF": { "estimators": ( diff --git a/test/test/sklearndf/transformation/test_transformation.py b/test/test/sklearndf/transformation/test_transformation.py index ab9484cf..cd1e4b41 100644 --- a/test/test/sklearndf/transformation/test_transformation.py +++ b/test/test/sklearndf/transformation/test_transformation.py @@ -1,12 +1,11 @@ -from typing import Type +from typing import Type, cast import numpy as np import pandas as pd - -# noinspection PyPackageRequirements import pytest import sklearn from pandas.util.testing import assert_frame_equal +from sklearn.base import BaseEstimator from sklearn.compose import ColumnTransformer from sklearn.preprocessing import Normalizer @@ -91,14 +90,15 @@ def test_special_wrapped_constructors() -> None: matching=r".*PowerTransformer|QuantileTransformer|.*Scaler", ), ) -def test_various_transformers(sklearn_cls: Type, test_data: pd.DataFrame) -> None: +def test_various_transformers( + sklearn_cls: Type[BaseEstimator], test_data: pd.DataFrame +) -> None: # get the wrapped counterpart for sklearn: - df_transf_cls = get_sklearndf_wrapper_class( + wrapper_class = get_sklearndf_wrapper_class( to_wrap=sklearn_cls, from_module=sklearndf.transformation ) - - # initalize both kind of transformers - df_t = df_transf_cls() + # initialize both kind of transformers + df_t = cast(TransformerDF, wrapper_class()) non_df_t = sklearn_cls() # for sklearn >=0.22 - check if not_fitted error is raised properly: