chore: apply linting changes

docs/source/conf.py

@@ -57,8 +57,8 @@
 
 # General information about the project.
 project = "Population Shift Monitoring"
-copyright = "2022, ING Bank N.V."
-author = "ING Wholesale Banking Advanced Analytics"
+copyright = "2023, ING Bank N.V."
+author = "ING Analytics Wholesale Banking"
 version = popmon.version.version
 
 # The language for content autogenerated by Sphinx. Refer to documentation
@@ -143,7 +143,7 @@
         master_doc,
         "popmon.tex",
         "POPMON Documentation",
-        "ING Wholesale Banking Advanced Analytics",
+        "ING Analytics Wholesale Banking",
         "manual",
     )
 ]

examples/integrations/kibana/popmon_to_elastic.py

@@ -4,7 +4,7 @@
 import pandas as pd
 from elastic_connector import ElasticConnector
 
-import popmon  # noqa
+import popmon  # noqa: F401
 from popmon import resources
 
 if __name__ == "__main__":

examples/synthetic_data.py

@@ -1,6 +1,6 @@
 import pandas as pd
 
-import popmon  # noqa
+import popmon  # noqa: F401
 from popmon import Settings, resources
 
 # open synthetic data

popmon/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2023 ING Analytics Wholesale Banking
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in

popmon/alerting/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2023 ING Analytics Wholesale Banking
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -18,8 +18,8 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
-from ..alerting.alerts_summary import AlertsSummary
-from ..alerting.compute_tl_bounds import (
+from popmon.alerting.alerts_summary import AlertsSummary
+from popmon.alerting.compute_tl_bounds import (
     ComputeTLBounds,
     DynamicBounds,
     StaticBounds,

popmon/alerting/alerts_summary.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2023 ING Analytics Wholesale Banking
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -24,7 +24,7 @@
 import numpy as np
 import pandas as pd
 
-from ..base import Module
+from popmon.base import Module
 
 
 class AlertsSummary(Module):

popmon/alerting/compute_tl_bounds.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2023 ING Analytics Wholesale Banking
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -26,8 +26,8 @@
 import numpy as np
 import pandas as pd
 
-from ..analysis.apply_func import ApplyFunc
-from ..base import Module, Pipeline
+from popmon.analysis.apply_func import ApplyFunc
+from popmon.base import Module, Pipeline
 
 
 def traffic_light_summary(row, cols=None, prefix=""):
@@ -95,7 +95,7 @@ def collect_traffic_light_bounds(monitoring_rules):
     """
     metrics_per_feature = defaultdict(list)
     metrics = []
-    for pattern in monitoring_rules.keys():
+    for pattern in monitoring_rules:
         psplit = pattern.split(":")
         feature = ":".join(psplit[:-1])
         metric = psplit[-1]

popmon/analysis/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2023 ING Analytics Wholesale Banking
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -18,7 +18,8 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
-from ..analysis.apply_func import ApplyFunc
+from popmon.analysis.apply_func import ApplyFunc
+
 from .comparison import Comparisons
 from .profiling import Profiles
 

popmon/analysis/apply_func.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2023 ING Analytics Wholesale Banking
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -24,8 +24,8 @@
 import numpy as np
 import pandas as pd
 
-from ..base import Module
-from ..utils import parallel
+from popmon.base import Module
+from popmon.utils import parallel
 
 
 class ApplyFunc(Module):
@@ -97,8 +97,8 @@ def add_apply_func(
         func,
         suffix=None,
         prefix=None,
-        metrics=[],
-        features=[],
+        metrics=None,
+        features=None,
         entire=None,
         *args,
         **kwargs,
@@ -117,6 +117,10 @@ def add_apply_func(
         :param kwargs: (dict, optional) kwargs for 'func'
         """
         # check inputs
+        if features is None:
+            features = []
+        if metrics is None:
+            metrics = []
         if not callable(func):
             raise TypeError("functions in ApplyFunc must be callable objects")
         if suffix is not None and not isinstance(suffix, str):
@@ -252,9 +256,12 @@ def apply_func(feature, selected_metrics, df, arr):
     :return: dictionary with outputs of applied-to metric pd.Series
     """
     # basic checks of feature
-    if "features" in arr and len(arr["features"]) > 0:
-        if feature not in arr["features"]:
-            return {}
+    if (
+        "features" in arr
+        and len(arr["features"]) > 0
+        and feature not in arr["features"]
+    ):
+        return {}
 
     # get func input
     keys = list(arr.keys())
@@ -343,20 +350,15 @@ def apply_func(feature, selected_metrics, df, arr):
         and all(obj.index == df.index)
     ):
         obj = {"_".join(df.columns): obj}
+    # e.g. output of normalized_hist_mean_cov: a dataframe with one column, actually a series
     elif (
         isinstance(obj, pd.DataFrame)
         and len(obj.columns) == 1
-        and len(obj.index) != len(df.index)
-    ):
-        # e.g. output of normalized_hist_mean_cov: a dataframe with one column, actually a series
-        obj = obj[obj.columns[0]].to_dict()
-    elif (
-        isinstance(obj, pd.DataFrame)
-        and len(obj.columns) == 1
-        and len(obj.index) == len(df.index)
-        and (obj.index != df.index).any()
+        and (
+            len(obj.index) != len(df.index)
+            or (len(obj.index) == len(df.index) and (obj.index != df.index).any())
+        )
     ):
-        # e.g. output of normalized_hist_mean_cov: a dataframe with one column, actually a series
         obj = obj[obj.columns[0]].to_dict()
     elif isinstance(obj, pd.Series):
         # e.g. output of np.mean of np.std: results in one number per column when applied to a dataframe

popmon/analysis/comparison/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2023 ING Analytics Wholesale Banking
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -16,14 +16,15 @@
 # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-from ...analysis.comparison.hist_comparer import (
+from popmon.analysis.comparison.hist_comparer import (
     ExpandingHistComparer,
     ExpandingNormHistComparer,
     ReferenceHistComparer,
     ReferenceNormHistComparer,
     RollingHistComparer,
     RollingNormHistComparer,
 )
+
 from .comparisons import Comparisons
 
 __all__ = [

popmon/analysis/comparison/comparisons.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2023 ING Analytics Wholesale Banking
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -82,7 +82,7 @@ def ks_test(hist_1, hist_2):
     Formulas translated from c++ to python, but formulas otherwise not modified.
     Reference: link: https://root.cern.ch/doc/master/classTH1.html#TH1:KolmogorovTest
     GNU license: https://root.cern.ch/license
-    All modifications copyright ING WBAA.
+    All modifications copyright INGA WB.
 
     :param hist_1: 1D array with bin counts of the histogram_1
     :param hist_2: 1D array with bin counts of the histogram_2
@@ -115,7 +115,7 @@ def ks_prob(testscore):
     Formulas translated from c++ to python, but formulas otherwise not modified.
     Reference: https://root.cern.ch/doc/master/classTH1.html#TH1:KolmogorovTest
     GNU license: https://root.cern.ch/license
-    All modifications copyright ING WBAA.
+    All modifications copyright INGA WB.
 
     :param float testscore: Kolmogorov-Smirnov test score
 
@@ -154,7 +154,7 @@ def ks_prob(testscore):
     dim=1,
     htype="num",
 )
-def ks(p, q, *args):
+def ks(p, q, *_):
     # KS-test only properly defined for (ordered) 1D interval variables
     ks_testscore = ks_test(p, q)
     ks_pvalue = ks_prob(ks_testscore)
@@ -182,7 +182,7 @@ def unknown_labels(hist1, hist2):
     dim=(2,),
     htype="all",
 )
-def pearson(p, q, *args):
+def pearson(p, q, *_):
     # calculate pearson coefficient
     pearson_coeff = np.nan
     if len(p) >= 2:
@@ -201,7 +201,7 @@ def uu_chi2(n, m):
     Formulas translated from c++ to python, but formulas otherwise not modified.
     Reference: https://root.cern.ch/doc/master/classTH1.html#a6c281eebc0c0a848e7a0d620425090a5
     GNU License: https://root.cern.ch/license
-    All modifications copyright ING WBAA.
+    All modifications copyright INGA WB.
 
     :param n: 1d array with bin counts of the reference set
     :param m: 1d array with bin counts of the test set
@@ -218,8 +218,8 @@ def _not_finite_to_zero(x):
     if len(n) != len(m):
         raise ValueError("Input histograms have unequal size.")
 
-    N = np.sum(n)
-    M = np.sum(m)
+    N = np.sum(n)  # noqa: N806
+    M = np.sum(m)  # noqa: N806
 
     if N == 0 or M == 0:
         return np.nan, np.nan, np.nan, np.nan, [0] * len(n)

popmon/analysis/comparison/hist_comparer.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2023 ING Analytics Wholesale Banking
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -21,8 +21,8 @@
 import numpy as np
 import pandas as pd
 
-from ...analysis.apply_func import ApplyFunc
-from ...analysis.functions import (
+from popmon.analysis.apply_func import ApplyFunc
+from popmon.analysis.functions import (
     expand_norm_hist_mean_cov,
     expanding_hist,
     hist_sum,
@@ -31,13 +31,13 @@
     roll_norm_hist_mean_cov,
     rolling_hist,
 )
-from ...analysis.hist_numpy import (
+from popmon.analysis.hist_numpy import (
     check_similar_hists,
     get_consistent_numpy_entries,
     get_consistent_numpy_ndgrids,
 )
-from ...base import Pipeline
-from ...hist.hist_utils import COMMON_HIST_TYPES, is_numeric
+from popmon.base import Pipeline
+from popmon.hist.hist_utils import COMMON_HIST_TYPES, is_numeric
 
 
 def hist_compare(row, hist_name1="", hist_name2=""):
@@ -132,7 +132,7 @@ def __init__(
             assign_to_key=assign_to_key,
         )
         hist_collector.add_apply_func(
-            func=func_hist_collector, entire=True, suffix=suffix, *args, **kwargs
+            *args, func=func_hist_collector, entire=True, suffix=suffix, **kwargs
         )
         # do histogram comparison
         hist_comparer = ApplyFunc(
@@ -326,7 +326,7 @@ def __init__(
         # make reference histogram(s)
         hist_collector = ApplyFunc(apply_to_key=read_key, assign_to_key=assign_to_key)
         hist_collector.add_apply_func(
-            func=func_hist_collector, hist_name=hist_col, suffix="", *args, **kwargs
+            *args, func=func_hist_collector, hist_name=hist_col, suffix="", **kwargs
         )
 
         # do histogram comparison

popmon/analysis/functions.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022 ING Wholesale Banking Advanced Analytics
+# Copyright (c) 2023 ING Analytics Wholesale Banking
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy of
 # this software and associated documentation files (the "Software"), to deal in
@@ -24,14 +24,14 @@
 from scipy import linalg, stats
 from scipy.stats import linregress, norm
 
-from ..analysis.hist_numpy import (
+from popmon.analysis.hist_numpy import (
     check_similar_hists,
     get_consistent_numpy_2dgrids,
     get_consistent_numpy_entries,
     set_2dgrid,
 )
-from ..hist.hist_utils import COMMON_HIST_TYPES, is_numeric
-from ..stats.numpy import probability_distribution_mean_covariance
+from popmon.hist.hist_utils import COMMON_HIST_TYPES, is_numeric
+from popmon.stats.numpy import probability_distribution_mean_covariance
 
 
 def pull(row, suffix_mean="_mean", suffix_std="_std", cols=None):
@@ -57,15 +57,17 @@ def pull(row, suffix_mean="_mean", suffix_std="_std", cols=None):
         ]
 
     x = {
-        m: np.nan
-        if (
-            any(
-                r not in rdict or pd.isnull(rdict[r])
-                for r in [m, m + suffix_mean, m + suffix_std]
+        m: (
+            np.nan
+            if (
+                any(
+                    r not in rdict or pd.isnull(rdict[r])
+                    for r in [m, m + suffix_mean, m + suffix_std]
+                )
+                or rdict[m + suffix_std] == 0.0
             )
-            or rdict[m + suffix_std] == 0.0
+            else (rdict[m] - rdict[m + suffix_mean]) / rdict[m + suffix_std]
         )
-        else (rdict[m] - rdict[m + suffix_mean]) / rdict[m + suffix_std]
         for m in cols
     }
 
@@ -183,6 +185,7 @@ def rolling_lr_zscore(df, window, shift=0):
     :param int shift: size of shift. default is 0.
     :return: df with rolling z-score results of lin_regress() function applied to all columns
     """
+
     # MB 20200420: turn original df.rolling off, it doesn't accept timestamps.
     # raw=True suppresses Future warning
     # return df.shift(shift).rolling(window).apply(func, raw=True)