diff --git a/src/evidently/base_metric.py b/src/evidently/base_metric.py index 268f541e84..83c42584ac 100644 --- a/src/evidently/base_metric.py +++ b/src/evidently/base_metric.py @@ -215,6 +215,11 @@ def result_type(cls) -> Type[MetricResult]: return typing_inspect.get_args(next(b for b in cls.__orig_bases__ if typing_inspect.is_generic_type(b)))[0] +class BasePreset(EvidentlyBaseModel): + class Config: + is_base_type = True + + class Metric(WithTestAndMetricDependencies, Generic[TResult], metaclass=WithResultFieldPathMetaclass): _context: Optional["Context"] = None diff --git a/src/evidently/features/generated_features.py b/src/evidently/features/generated_features.py index 68350f3cf7..19e15ac3b6 100644 --- a/src/evidently/features/generated_features.py +++ b/src/evidently/features/generated_features.py @@ -208,6 +208,9 @@ def on(self, columns: List[str]) -> "ColumnName": class FeatureDescriptor(EvidentlyBaseModel): + class Config: + is_base_type = True + display_name: Optional[str] = None def for_column(self, column_name: str) -> "ColumnName": diff --git a/src/evidently/metric_preset/classification_performance.py b/src/evidently/metric_preset/classification_performance.py index ddd0203a92..4e1b79c25d 100644 --- a/src/evidently/metric_preset/classification_performance.py +++ b/src/evidently/metric_preset/classification_performance.py @@ -3,6 +3,7 @@ from typing import List from typing import Optional +from evidently.metric_preset.metric_preset import AnyMetric from evidently.metric_preset.metric_preset import MetricPreset from evidently.metrics import ClassificationClassBalance from evidently.metrics import ClassificationClassSeparationPlot @@ -35,13 +36,15 @@ class ClassificationPreset(MetricPreset): def __init__( self, columns: Optional[List[str]] = None, probas_threshold: Optional[float] = None, k: Optional[int] = None ): - super().__init__() self.columns = columns self.probas_threshold = probas_threshold self.k = k + super().__init__() - def generate_metrics(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): - result = [ + def generate_metrics( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyMetric]: + result: List[AnyMetric] = [ ClassificationQualityMetric(probas_threshold=self.probas_threshold, k=self.k), ClassificationClassBalance(), ClassificationConfusionMatrix(probas_threshold=self.probas_threshold, k=self.k), diff --git a/src/evidently/metric_preset/data_drift.py b/src/evidently/metric_preset/data_drift.py index 9b4204076f..fbb577e950 100644 --- a/src/evidently/metric_preset/data_drift.py +++ b/src/evidently/metric_preset/data_drift.py @@ -4,6 +4,7 @@ from typing import Optional from evidently.calculations.stattests import PossibleStatTestType +from evidently.metric_preset.metric_preset import AnyMetric from evidently.metric_preset.metric_preset import MetricPreset from evidently.metrics import DataDriftTable from evidently.metrics import DatasetDriftMetric @@ -54,7 +55,6 @@ def __init__( text_stattest_threshold: Optional[float] = None, per_column_stattest_threshold: Optional[Dict[str, float]] = None, ): - super().__init__() self.columns = columns self.embeddings = embeddings self.embeddings_drift_method = embeddings_drift_method @@ -69,9 +69,12 @@ def __init__( self.num_stattest_threshold = num_stattest_threshold self.text_stattest_threshold = text_stattest_threshold self.per_column_stattest_threshold = per_column_stattest_threshold + super().__init__() - def generate_metrics(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): - result = [ + def generate_metrics( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyMetric]: + result: List[AnyMetric] = [ DatasetDriftMetric( columns=self.columns, drift_share=self.drift_share, @@ -103,7 +106,7 @@ def generate_metrics(self, data_definition: DataDefinition, additional_data: Opt embeddings_data = data_definition.embeddings if embeddings_data is None: return result - result = add_emb_drift_to_reports( - result, embeddings_data, self.embeddings, self.embeddings_drift_method, EmbeddingsDriftMetric + result += add_emb_drift_to_reports( + embeddings_data, self.embeddings, self.embeddings_drift_method, EmbeddingsDriftMetric ) return result diff --git a/src/evidently/metric_preset/data_quality.py b/src/evidently/metric_preset/data_quality.py index 98eedc710e..c536b3de44 100644 --- a/src/evidently/metric_preset/data_quality.py +++ b/src/evidently/metric_preset/data_quality.py @@ -3,6 +3,7 @@ from typing import List from typing import Optional +from evidently.metric_preset.metric_preset import AnyMetric from evidently.metric_preset.metric_preset import MetricPreset from evidently.metrics import ColumnSummaryMetric from evidently.metrics import DatasetSummaryMetric @@ -27,10 +28,12 @@ class DataQualityPreset(MetricPreset): columns: Optional[List[str]] def __init__(self, columns: Optional[List[str]] = None): - super().__init__() self.columns = columns + super().__init__() - def generate_metrics(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_metrics( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyMetric]: return [ DatasetSummaryMetric(), generate_column_metrics(ColumnSummaryMetric, columns=self.columns, skip_id_column=True), diff --git a/src/evidently/metric_preset/metric_preset.py b/src/evidently/metric_preset/metric_preset.py index 272e25f679..d939c8bd87 100644 --- a/src/evidently/metric_preset/metric_preset.py +++ b/src/evidently/metric_preset/metric_preset.py @@ -1,17 +1,23 @@ import abc from typing import Any from typing import Dict +from typing import List from typing import Optional +from typing import Union +from evidently.base_metric import BasePreset +from evidently.base_metric import Metric from evidently.utils.data_preprocessing import DataDefinition +from evidently.utils.generators import BaseGenerator +AnyMetric = Union[Metric, BaseGenerator[Metric]] -class MetricPreset: - """Base class for metric presets""" - def __init__(self): - pass +class MetricPreset(BasePreset): + """Base class for metric presets""" @abc.abstractmethod - def generate_metrics(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_metrics( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyMetric]: raise NotImplementedError() diff --git a/src/evidently/metric_preset/recsys.py b/src/evidently/metric_preset/recsys.py index 2aa5a364e5..b11f364507 100644 --- a/src/evidently/metric_preset/recsys.py +++ b/src/evidently/metric_preset/recsys.py @@ -4,6 +4,7 @@ from typing import Optional from typing import Union +from evidently.metric_preset.metric_preset import AnyMetric from evidently.metric_preset.metric_preset import MetricPreset from evidently.metrics import DiversityMetric from evidently.metrics import FBetaTopKMetric @@ -69,7 +70,6 @@ def __init__( user_bias_columns: Optional[List[str]] = None, item_bias_columns: Optional[List[str]] = None, ): - super().__init__() self.k = k self.min_rel_score = min_rel_score self.no_feedback_users = no_feedback_users @@ -79,12 +79,15 @@ def __init__( self.item_features = item_features self.user_bias_columns = user_bias_columns self.item_bias_columns = item_bias_columns + super().__init__() - def generate_metrics(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_metrics( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyMetric]: is_train_data = False if additional_data is not None: is_train_data = "current_train_data" in additional_data.keys() - metrics = [ + metrics: List[AnyMetric] = [ PrecisionTopKMetric(k=self.k, min_rel_score=self.min_rel_score, no_feedback_users=self.no_feedback_users), RecallTopKMetric(k=self.k, min_rel_score=self.min_rel_score, no_feedback_users=self.no_feedback_users), FBetaTopKMetric(k=self.k, min_rel_score=self.min_rel_score, no_feedback_users=self.no_feedback_users), diff --git a/src/evidently/metric_preset/regression_performance.py b/src/evidently/metric_preset/regression_performance.py index 31c8002a43..4414cacc67 100644 --- a/src/evidently/metric_preset/regression_performance.py +++ b/src/evidently/metric_preset/regression_performance.py @@ -3,6 +3,7 @@ from typing import List from typing import Optional +from evidently.metric_preset.metric_preset import AnyMetric from evidently.metric_preset.metric_preset import MetricPreset from evidently.metrics import RegressionAbsPercentageErrorPlot from evidently.metrics import RegressionErrorBiasTable @@ -34,10 +35,12 @@ class RegressionPreset(MetricPreset): columns: Optional[List[str]] def __init__(self, columns: Optional[List[str]] = None): - super().__init__() self.columns = columns + super().__init__() - def generate_metrics(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_metrics( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyMetric]: return [ RegressionQualityMetric(), RegressionPredictedVsActualScatter(), diff --git a/src/evidently/metric_preset/target_drift.py b/src/evidently/metric_preset/target_drift.py index dd24e2be66..713951400c 100644 --- a/src/evidently/metric_preset/target_drift.py +++ b/src/evidently/metric_preset/target_drift.py @@ -2,10 +2,9 @@ from typing import Dict from typing import List from typing import Optional -from typing import Sequence -from evidently.base_metric import Metric from evidently.calculations.stattests import PossibleStatTestType +from evidently.metric_preset.metric_preset import AnyMetric from evidently.metric_preset.metric_preset import MetricPreset from evidently.metrics import ColumnCorrelationsMetric from evidently.metrics import ColumnDriftMetric @@ -52,7 +51,6 @@ def __init__( text_stattest_threshold: Optional[float] = None, per_column_stattest_threshold: Optional[Dict[str, float]] = None, ): - super().__init__() self.columns = columns self.stattest = stattest self.cat_stattest = cat_stattest @@ -64,13 +62,14 @@ def __init__( self.num_stattest_threshold = num_stattest_threshold self.text_stattest_threshold = text_stattest_threshold self.per_column_stattest_threshold = per_column_stattest_threshold + super().__init__() def generate_metrics( self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] - ) -> Sequence[Metric]: + ) -> List[AnyMetric]: target = data_definition.get_target_column() prediction = data_definition.get_prediction_columns() - result: List[Metric] = [] + result: List[AnyMetric] = [] columns_by_target = [] if target is not None: diff --git a/src/evidently/metric_preset/text_evals.py b/src/evidently/metric_preset/text_evals.py index 3525a23bd0..9bc134f474 100644 --- a/src/evidently/metric_preset/text_evals.py +++ b/src/evidently/metric_preset/text_evals.py @@ -9,18 +9,24 @@ from evidently.descriptors import Sentiment from evidently.descriptors import TextLength from evidently.features.generated_features import FeatureDescriptor +from evidently.metric_preset.metric_preset import AnyMetric from evidently.metric_preset.metric_preset import MetricPreset from evidently.metrics import ColumnSummaryMetric from evidently.utils.data_preprocessing import DataDefinition class TextEvals(MetricPreset): + column_name: str + descriptors: Optional[List[FeatureDescriptor]] = None + def __init__(self, column_name: str, descriptors: Optional[List[FeatureDescriptor]] = None): - super().__init__() self.column_name: str = column_name self.descriptors: Optional[List[FeatureDescriptor]] = descriptors + super().__init__() - def generate_metrics(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_metrics( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyMetric]: descriptors = self.descriptors or [ TextLength(), SentenceCount(), diff --git a/src/evidently/metric_preset/text_overview.py b/src/evidently/metric_preset/text_overview.py index 22ac23aef9..a646d7cf67 100644 --- a/src/evidently/metric_preset/text_overview.py +++ b/src/evidently/metric_preset/text_overview.py @@ -10,6 +10,7 @@ from evidently.descriptors import Sentiment from evidently.descriptors import TextLength from evidently.features.generated_features import FeatureDescriptor +from evidently.metric_preset.metric_preset import AnyMetric from evidently.metric_preset.metric_preset import MetricPreset from evidently.metrics import ColumnSummaryMetric from evidently.utils.data_preprocessing import DataDefinition @@ -30,6 +31,7 @@ class TextOverviewPreset(MetricPreset): """ columns: List[str] + descriptors: Optional[List[FeatureDescriptor]] = None def __init__( self, @@ -37,7 +39,6 @@ def __init__( columns: Optional[List[str]] = None, descriptors: Optional[List[FeatureDescriptor]] = None, ): - super().__init__() if column_name is not None and columns is not None: raise ValueError("Cannot specify both `columns` and `columns`.") if columns is not None: @@ -47,9 +48,12 @@ def __init__( else: raise ValueError("Must specify either `columns` or `columns`.") self.descriptors = descriptors + super().__init__() - def generate_metrics(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): - result = [] + def generate_metrics( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyMetric]: + result: List[AnyMetric] = [] if self.descriptors is None: descriptors = [ TextLength(), diff --git a/src/evidently/metrics/base_metric.py b/src/evidently/metrics/base_metric.py index 54d52075c4..6917110a13 100644 --- a/src/evidently/metrics/base_metric.py +++ b/src/evidently/metrics/base_metric.py @@ -13,7 +13,7 @@ def generate_column_metrics( columns: Optional[Union[str, list]] = None, parameters: Optional[Dict] = None, skip_id_column: bool = False, -) -> BaseGenerator: +) -> BaseGenerator[Metric]: """Function for generating metrics for columns""" return make_generator_by_columns( base_class=metric_class, diff --git a/src/evidently/report/report.py b/src/evidently/report/report.py index 342c297cb9..2f61369a62 100644 --- a/src/evidently/report/report.py +++ b/src/evidently/report/report.py @@ -135,7 +135,7 @@ def run( self.metadata[METRIC_GENERATORS] = [] self.metadata[METRIC_GENERATORS].append(item.__class__.__name__) # type: ignore[union-attr] elif isinstance(item, MetricPreset): - metrics = [] + metrics: List[Metric] = [] for metric_item in item.generate_metrics(data_definition, additional_data=additional_data): if isinstance(metric_item, BaseGenerator): diff --git a/src/evidently/test_preset/classification_binary.py b/src/evidently/test_preset/classification_binary.py index 76b479bb72..928c6a0d77 100644 --- a/src/evidently/test_preset/classification_binary.py +++ b/src/evidently/test_preset/classification_binary.py @@ -1,8 +1,10 @@ from typing import Any from typing import Dict +from typing import List from typing import Optional from evidently.calculations.stattests import PossibleStatTestType +from evidently.test_preset.test_preset import AnyTest from evidently.test_preset.test_preset import TestPreset from evidently.tests import TestAccuracyScore from evidently.tests import TestColumnDrift @@ -29,18 +31,24 @@ class BinaryClassificationTestPreset(TestPreset): - `TestAccuracyScore` """ + stattest: Optional[PossibleStatTestType] = None + stattest_threshold: Optional[float] = None + probas_threshold: Optional[float] = None + def __init__( self, stattest: Optional[PossibleStatTestType] = None, stattest_threshold: Optional[float] = None, probas_threshold: Optional[float] = None, ): - super().__init__() self.stattest = stattest self.stattest_threshold = stattest_threshold self.probas_threshold = probas_threshold + super().__init__() - def generate_tests(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_tests( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyTest]: target = data_definition.get_target_column() if target is None: diff --git a/src/evidently/test_preset/classification_binary_topk.py b/src/evidently/test_preset/classification_binary_topk.py index 14fc505eaa..f5ff3a4523 100644 --- a/src/evidently/test_preset/classification_binary_topk.py +++ b/src/evidently/test_preset/classification_binary_topk.py @@ -1,8 +1,10 @@ from typing import Any from typing import Dict +from typing import List from typing import Optional from evidently.calculations.stattests import PossibleStatTestType +from evidently.test_preset.test_preset import AnyTest from evidently.test_preset.test_preset import TestPreset from evidently.tests import TestAccuracyScore from evidently.tests import TestColumnDrift @@ -29,6 +31,7 @@ class BinaryClassificationTopKTestPreset(TestPreset): - `TestAccuracyScore` """ + k: int stattest: Optional[PossibleStatTestType] stattest_threshold: Optional[float] @@ -38,12 +41,14 @@ def __init__( stattest: Optional[PossibleStatTestType] = None, stattest_threshold: Optional[float] = None, ): - super().__init__() self.k = k self.stattest = stattest self.stattest_threshold = stattest_threshold + super().__init__() - def generate_tests(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_tests( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyTest]: target = data_definition.get_target_column() if target is None: raise ValueError("Target column should be set in mapping and be present in data") diff --git a/src/evidently/test_preset/classification_multiclass.py b/src/evidently/test_preset/classification_multiclass.py index b3566ffcab..45ccb34aed 100644 --- a/src/evidently/test_preset/classification_multiclass.py +++ b/src/evidently/test_preset/classification_multiclass.py @@ -1,8 +1,10 @@ from typing import Any from typing import Dict +from typing import List from typing import Optional from evidently.calculations.stattests import PossibleStatTestType +from evidently.test_preset.test_preset import AnyTest from evidently.test_preset.test_preset import TestPreset from evidently.tests import TestAccuracyScore from evidently.tests import TestColumnDrift @@ -42,12 +44,13 @@ def __init__( stattest: Optional[PossibleStatTestType] = None, stattest_threshold: Optional[float] = None, ): - super().__init__() - self.stattest = stattest self.stattest_threshold = stattest_threshold + super().__init__() - def generate_tests(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_tests( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyTest]: target = data_definition.get_target_column() if target is None: @@ -61,7 +64,7 @@ def generate_tests(self, data_definition: DataDefinition, additional_data: Optio classification_labels if isinstance(classification_labels, list) else classification_labels.values() ) - tests = [ + tests: List[AnyTest] = [ TestAccuracyScore(), TestF1Score(), *[TestPrecisionByClass(label) for label in labels], diff --git a/src/evidently/test_preset/data_drift.py b/src/evidently/test_preset/data_drift.py index 0ea76f9aca..bccc0b5b25 100644 --- a/src/evidently/test_preset/data_drift.py +++ b/src/evidently/test_preset/data_drift.py @@ -8,6 +8,7 @@ from evidently.calculations.stattests import PossibleStatTestType from evidently.metrics.data_drift.embedding_drift_methods import DriftMethod from evidently.pipeline.column_mapping import TaskType +from evidently.test_preset.test_preset import AnyTest from evidently.test_preset.test_preset import TestPreset from evidently.tests import TestAllFeaturesValueDrift from evidently.tests import TestColumnDrift @@ -61,7 +62,6 @@ def __init__( text_stattest_threshold: Optional[float] = None, per_column_stattest_threshold: Optional[Dict[str, float]] = None, ): - super().__init__() self.columns = columns self.embeddings = embeddings self.embeddings_drift_method = embeddings_drift_method @@ -76,8 +76,11 @@ def __init__( self.num_stattest_threshold = num_stattest_threshold self.text_stattest_threshold = text_stattest_threshold self.per_column_stattest_threshold = per_column_stattest_threshold + super().__init__() - def generate_tests(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_tests( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyTest]: embeddings_data = data_definition.embeddings if embeddings_data is not None: embs = list(set(v for values in embeddings_data.values() for v in values)) @@ -174,8 +177,7 @@ def generate_tests(self, data_definition: DataDefinition, additional_data: Optio if embeddings_data is None: return preset_tests - preset_tests = add_emb_drift_to_reports( - preset_tests, + preset_tests += add_emb_drift_to_reports( embeddings_data, self.embeddings, self.embeddings_drift_method, diff --git a/src/evidently/test_preset/data_quality.py b/src/evidently/test_preset/data_quality.py index f4f083b763..1e03eddd37 100644 --- a/src/evidently/test_preset/data_quality.py +++ b/src/evidently/test_preset/data_quality.py @@ -3,6 +3,7 @@ from typing import List from typing import Optional +from evidently.test_preset.test_preset import AnyTest from evidently.test_preset.test_preset import TestPreset from evidently.tests import TestAllColumnsMostCommonValueShare from evidently.tests import TestAllColumnsShareOfMissingValues @@ -31,10 +32,12 @@ def __init__( self, columns: Optional[List[str]] = None, ): - super().__init__() self.columns = columns + super().__init__() - def generate_tests(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_tests( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyTest]: return [ TestAllColumnsShareOfMissingValues(columns=self.columns), TestAllColumnsMostCommonValueShare(columns=self.columns), diff --git a/src/evidently/test_preset/data_stability.py b/src/evidently/test_preset/data_stability.py index 89a7194a2f..5c4e384778 100644 --- a/src/evidently/test_preset/data_stability.py +++ b/src/evidently/test_preset/data_stability.py @@ -3,6 +3,7 @@ from typing import List from typing import Optional +from evidently.test_preset.test_preset import AnyTest from evidently.test_preset.test_preset import TestPreset from evidently.tests import TestAllColumnsShareOfMissingValues from evidently.tests import TestCatColumnsOutOfListValues @@ -34,10 +35,12 @@ def __init__( self, columns: Optional[List[str]] = None, ): - super().__init__() self.columns = columns + super().__init__() - def generate_tests(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_tests( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyTest]: return [ TestNumberOfRows(), TestNumberOfColumns(), diff --git a/src/evidently/test_preset/no_target_performance.py b/src/evidently/test_preset/no_target_performance.py index 1981d10dd3..e88bc58106 100644 --- a/src/evidently/test_preset/no_target_performance.py +++ b/src/evidently/test_preset/no_target_performance.py @@ -8,6 +8,7 @@ from evidently.calculations.stattests import PossibleStatTestType from evidently.metrics.data_drift.embedding_drift_methods import DriftMethod from evidently.pipeline.column_mapping import TaskType +from evidently.test_preset.test_preset import AnyTest from evidently.test_preset.test_preset import TestPreset from evidently.tests import TestAllColumnsShareOfMissingValues from evidently.tests import TestCatColumnsOutOfListValues @@ -73,7 +74,6 @@ def __init__( text_stattest_threshold: Optional[float] = None, per_column_stattest_threshold: Optional[Dict[str, float]] = None, ): - super().__init__() self.columns = columns self.embeddings = embeddings self.embeddings_drift_method = embeddings_drift_method @@ -87,9 +87,12 @@ def __init__( self.cat_stattest_threshold = cat_stattest_threshold self.num_stattest_threshold = num_stattest_threshold self.text_stattest_threshold = text_stattest_threshold - self.per_feature_threshold = per_column_stattest_threshold + self.per_column_stattest_threshold = per_column_stattest_threshold + super().__init__() - def generate_tests(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_tests( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyTest]: embeddings_data = data_definition.embeddings if embeddings_data is not None: embs = list(set(v for values in embeddings_data.values() for v in values)) @@ -140,7 +143,7 @@ def generate_tests(self, data_definition: DataDefinition, additional_data: Optio cat_stattest_threshold=self.cat_stattest_threshold, num_stattest_threshold=self.num_stattest_threshold, text_stattest_threshold=self.text_stattest_threshold, - per_column_stattest_threshold=self.per_feature_threshold, + per_column_stattest_threshold=self.per_column_stattest_threshold, ) ) preset_tests.append(TestColumnsType()) @@ -151,8 +154,7 @@ def generate_tests(self, data_definition: DataDefinition, additional_data: Optio if embeddings_data is None: return preset_tests - preset_tests = add_emb_drift_to_reports( - preset_tests, + preset_tests += add_emb_drift_to_reports( embeddings_data, self.embeddings, self.embeddings_drift_method, diff --git a/src/evidently/test_preset/recsys.py b/src/evidently/test_preset/recsys.py index f23069a83a..e5ebd78fff 100644 --- a/src/evidently/test_preset/recsys.py +++ b/src/evidently/test_preset/recsys.py @@ -1,7 +1,9 @@ from typing import Any from typing import Dict +from typing import List from typing import Optional +from evidently.test_preset.test_preset import AnyTest from evidently.test_preset.test_preset import TestPreset from evidently.tests import TestHitRateK from evidently.tests import TestMAPK @@ -28,12 +30,14 @@ class RecsysTestPreset(TestPreset): no_feedback_users: bool def __init__(self, k: int, min_rel_score: Optional[int] = None, no_feedback_users: bool = False): - super().__init__() self.k = k self.min_rel_score = min_rel_score self.no_feedback_users = no_feedback_users + super().__init__() - def generate_tests(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_tests( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyTest]: return [ TestPrecisionTopK(k=self.k, min_rel_score=self.min_rel_score, no_feedback_users=self.no_feedback_users), TestRecallTopK(k=self.k, min_rel_score=self.min_rel_score, no_feedback_users=self.no_feedback_users), diff --git a/src/evidently/test_preset/regression.py b/src/evidently/test_preset/regression.py index cb84b65011..518900ac65 100644 --- a/src/evidently/test_preset/regression.py +++ b/src/evidently/test_preset/regression.py @@ -1,7 +1,9 @@ from typing import Any from typing import Dict +from typing import List from typing import Optional +from evidently.test_preset.test_preset import AnyTest from evidently.test_preset.test_preset import TestPreset from evidently.tests import TestValueMAE from evidently.tests import TestValueMAPE @@ -21,7 +23,9 @@ class RegressionTestPreset(TestPreset): - `TestValueMAPE` """ - def generate_tests(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): + def generate_tests( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyTest]: return [ TestValueMeanError(), TestValueMAE(), diff --git a/src/evidently/test_preset/test_preset.py b/src/evidently/test_preset/test_preset.py index f820c69c72..7ece7f5eef 100644 --- a/src/evidently/test_preset/test_preset.py +++ b/src/evidently/test_preset/test_preset.py @@ -1,15 +1,21 @@ import abc from typing import Any from typing import Dict +from typing import List from typing import Optional +from typing import Union +from evidently.base_metric import BasePreset +from evidently.tests.base_test import Test from evidently.utils.data_preprocessing import DataDefinition +from evidently.utils.generators import BaseGenerator +AnyTest = Union[Test, BaseGenerator[Test]] -class TestPreset: - def __init__(self): - pass +class TestPreset(BasePreset): @abc.abstractmethod - def generate_tests(self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]]): - raise NotImplementedError() + def generate_tests( + self, data_definition: DataDefinition, additional_data: Optional[Dict[str, Any]] + ) -> List[AnyTest]: + raise NotImplementedError diff --git a/src/evidently/utils/data_drift_utils.py b/src/evidently/utils/data_drift_utils.py index da3e81046e..8fcceceb48 100644 --- a/src/evidently/utils/data_drift_utils.py +++ b/src/evidently/utils/data_drift_utils.py @@ -12,6 +12,7 @@ from sklearn.model_selection import train_test_split from sklearn.pipeline import Pipeline +from evidently.base_metric import Metric from evidently.calculations.stattests import PossibleStatTestType @@ -231,12 +232,12 @@ def get_text_data_for_plots(reference_data: pd.Series, current_data: pd.Series): def add_emb_drift_to_reports( - result, embeddings_data, embeddings, embeddings_drift_method, entity, -): +) -> List[Metric]: + result: List[Metric] = [] sets = list(embeddings_data.keys()) if embeddings is not None: sets = np.intersect1d(sets, embeddings) diff --git a/src/evidently/utils/generators.py b/src/evidently/utils/generators.py index e2481779c8..8353857d07 100644 --- a/src/evidently/utils/generators.py +++ b/src/evidently/utils/generators.py @@ -44,11 +44,11 @@ def generate(self, data_definition: DataDefinition) -> List[TObject]: def make_generator_by_columns( - base_class: Type, + base_class: Type[TObject], columns: Optional[Union[str, list]] = None, parameters: Optional[Dict] = None, skip_id_column: bool = False, -) -> BaseGenerator: +) -> BaseGenerator[TObject]: """Create a test generator for a columns list with a test class. Base class is specified with `base_class` parameter. diff --git a/ui/packages/evidently-ui-lib/src/components/ProjectCard.tsx b/ui/packages/evidently-ui-lib/src/components/ProjectCard.tsx index 81b1a530b4..8305e41fa4 100644 --- a/ui/packages/evidently-ui-lib/src/components/ProjectCard.tsx +++ b/ui/packages/evidently-ui-lib/src/components/ProjectCard.tsx @@ -74,6 +74,7 @@ export const EditProjectInfoForm = ({ // to project object, then it goes to the action submit( + // @ts-ignore { ...project, name,