Feature: add List and Frequent Items Comparison Operators (#31)

* Adding FrequentItems and ListComparison
* Adding ComparisonConfig

examples/presets.md

@@ -122,6 +122,49 @@ manager = MonitorManager(setup=monitor_setup)
 manager.save()
 ```
 
+### List Comparison
+
+```python
+from whylabs_toolkit.monitor import MonitorSetup, MonitorManager
+from whylabs_toolkit.monitor.models import *
+
+setup = MonitorSetup(monitor_id="monitor_list_comparison")
+setup.config = ListComparisonConfig(
+ operator=ListComparisonOperator.in_list,
+ expected=[
+ ExpectedValue(
+ str="expected"
+ ),
+ ExpectedValue(
+ int=123229
+ )
+ ],
+ baseline=TrailingWindowBaseline(size=7),
+ metric=SimpleColumnMetric.count_bool
+)
+setup.apply()
+
+mm = MonitorManager(setup=setup)
+mm.save()
+```
+
+### Frequent Items
+```python
+from whylabs_toolkit.monitor import MonitorSetup, MonitorManager
+from whylabs_toolkit.monitor.models import *
+
+
+setup = MonitorSetup(monitor_id="frequent_items")
+setup.config = FrequentStringComparisonConfig(
+ operator=FrequentStringComparisonOperator.eq,
+ baseline=TrailingWindowBaseline(size=7)
+)
+setup.apply()
+
+mm = MonitorManager(setup=setup)
+mm.save()
+```
+
 ## Model Performance
 
 ### F1 Score

whylabs_toolkit/monitor/manager/monitor_setup.py

@@ -36,7 +36,10 @@ def __init__(self, monitor_id: str, dataset_id: Optional[str] = None, config: Co
  FixedThresholdsConfig,
  StddevConfig,
  DriftConfig,
+ ComparisonConfig,
  SeasonalConfig,
+ FrequentStringComparisonConfig,
+ ListComparisonConfig,
  ]
  ] = None
  self._target_columns: Optional[List[str]] = []
@@ -100,7 +103,18 @@ def target_matrix(self, target: Union[ColumnMatrix, DatasetMatrix]) -> None:
  @property
  def config(
  self,
- ) -> Optional[Union[DiffConfig, FixedThresholdsConfig, StddevConfig, DriftConfig, SeasonalConfig,]]:
+ ) -> Optional[
+ Union[
+ DiffConfig,
+ FixedThresholdsConfig,
+ StddevConfig,
+ DriftConfig,
+ ComparisonConfig,
+ SeasonalConfig,
+ FrequentStringComparisonConfig,
+ ListComparisonConfig,
+ ]
+ ]:
  return self._analyzer_config
 
  @config.setter
@@ -112,6 +126,8 @@ def config(
  StddevConfig,
  DriftConfig,
  SeasonalConfig,
+ FrequentStringComparisonConfig,
+ ListComparisonConfig,
  ],
  ) -> None:
  self._analyzer_config = config

whylabs_toolkit/monitor/models/__init__.py

@@ -30,6 +30,10 @@
  "DriftConfig",
  "ComparisonConfig",
  "ComparisonOperator",
+ "FrequentStringComparisonConfig",
+ "FrequentStringComparisonOperator",
+ "ListComparisonOperator",
+ "ListComparisonConfig",
  "ExperimentalConfig",
  "FixedThresholdsConfig",
  "ColumnListChangeConfig",
@@ -73,4 +77,5 @@
  "DatasetMetric",
  "SimpleColumnMetric",
  "ComplexMetrics",
+ "ExpectedValue",
 ]

whylabs_toolkit/monitor/models/analyzer/__init__.py

@@ -26,6 +26,8 @@
  "FixedThresholdsConfig",
  "ColumnListChangeConfig",
  "SeasonalConfig",
+ "ListComparisonConfig",
+ "FrequentStringComparisonConfig",
  "StddevConfig",
  # enums
  "DiffMode",
@@ -34,8 +36,11 @@
  "DatasetMetric",
  "SimpleColumnMetric",
  "ComplexMetrics",
+ "ListComparisonOperator",
+ "FrequentStringComparisonOperator",
  # targets
  "DatasetMatrix",
  "ColumnMatrix",
  "TargetLevel",
+ "ExpectedValue",
 ]

whylabs_toolkit/monitor/models/analyzer/algorithms.py

@@ -20,6 +20,8 @@ class AlgorithmType(str, Enum):
  expected = "expected"
  column_list = "column_list"
  comparison = "comparison"
+ list_comparison = "list_comparison"
+ frequent_string_comparison = "frequent_string_comparison"
  diff = "diff"
  drift = "drift"
  stddev = "stddev"
@@ -148,13 +150,26 @@ class ComparisonOperator(str, Enum):
  """Operators for performing a comparison."""
 
  eq = "eq"
+ # Not Yet Implemented:
+ # gt = "gt"
+ # lt = "lt"
+ # ge = "ge"
+ # le = "le"
 
 
-# Not Yet Implemented:
-# gt = "gt"
-# lt = "lt"
-# ge = "ge"
-# le = "le"
+class ListComparisonOperator(str, Enum):
+ """Operators for performing a comparison."""
+
+ in_list = "in"
+ not_in_list = "not_in"
+
+
+class FrequentStringComparisonOperator(str, Enum):
+ """Operators for performing a comparison."""
+
+ eq = "eq"
+ target_includes_all_baseline = "target_includes_all_baseline"
+ baseline_includes_all_target = "baseline_includes_all_target"
 
 
 class ComparisonConfig(AlgorithmConfig):
@@ -248,9 +263,7 @@ class SeasonalConfig(_ThresholdBaseConfig):
  """
 
  type: Literal[AlgorithmType.seasonal] = AlgorithmType.seasonal
- algorithm: Literal["arima", "rego", "stastforecast"] = Field(
- "arima", description="The algorithm implementation for seasonal analysis"
- )
+ algorithm: Literal["arima"] = Field("arima", description="The algorithm implementation for seasonal analysis")
  minBatchSize: Optional[int] = Field(
  30,
  title="MinBatchSize",
@@ -287,7 +300,7 @@ class DriftConfig(AlgorithmConfig):
  """
 
  type: Literal[AlgorithmType.drift] = AlgorithmType.drift
- algorithm: Literal["hellinger", "ks_test", "kl_divergence", "variation_distance"] = Field(
+ algorithm: Literal["hellinger", "jensenshannon", "kl_divergence", "psi"] = Field(
  "hellinger", description="The algorithm to use when calculating drift."
  )
  metric: Literal[ComplexMetrics.histogram, ComplexMetrics.frequent_items]
@@ -314,6 +327,31 @@ class ExperimentalConfig(AlgorithmConfig):
  stub: Optional[AlgorithmType] = Field(description="Stub field to flow algoirthm type into the schema. Do not use.")
 
 
+class ListComparisonConfig(AlgorithmConfig):
+ """Compare a target list of values against a baseline list of values."""
+
+ type: Literal[AlgorithmType.list_comparison] = AlgorithmType.list_comparison
+ operator: ListComparisonOperator = Field(
+ description="The operator for the comparison. The right hand side is the target batch's metric. The left hand"
+ "side is the expected value or a baseline's metric."
+ )
+ expected: Optional[List[ExpectedValue]] = Field(
+ None,
+ description="The expected values of the equality. If the value is not set we will extract the corresponding "
+ "metric from the baseline and perform the comparison",
+ )
+ baseline: Optional[Union[TrailingWindowBaseline, ReferenceProfileId, TimeRangeBaseline, SingleBatchBaseline]]
+
+
+class FrequentStringComparisonConfig(AlgorithmConfig):
+ """Compare whether target against a list of values."""
+
+ type: Literal[AlgorithmType.frequent_string_comparison] = AlgorithmType.frequent_string_comparison
+ metric: Literal[ComplexMetrics.frequent_items] = ComplexMetrics.frequent_items
+ operator: FrequentStringComparisonOperator = Field(description="The operator for the comparison.")
+ baseline: Union[TrailingWindowBaseline, ReferenceProfileId, TimeRangeBaseline, SingleBatchBaseline]
+
+
 class DiffMode(str, Enum):
  """Whether to use the absolute difference or the percentage to calculate the difference."""
 

whylabs_toolkit/monitor/models/analyzer/analyzer.py

@@ -10,6 +10,8 @@
 from .algorithms import (
  ColumnListChangeConfig,
  ComparisonConfig,
+ ListComparisonConfig,
+ FrequentStringComparisonConfig,
  DiffConfig,
  DriftConfig,
  ExperimentalConfig,
@@ -90,15 +92,17 @@ class Analyzer(NoExtrasBaseModel):
  )
 
  # NOT YET IMPLEMENTED:
- # ComparisonConfig,
  # ExperimentalConfig,
  # ColumnListChangeConfig,
 
  config: Union[
  DiffConfig,
  FixedThresholdsConfig,
+ ListComparisonConfig,
+ FrequentStringComparisonConfig,
  StddevConfig,
  DriftConfig,
+ ComparisonConfig,
  SeasonalConfig,
  ] = Field(description="The configuration map of the analyzer", discriminator="type")