feat: added field type inside drift data (#142)

radicalbit · Jul 26, 2024 · a19cd20 · a19cd20
1 parent b08e97d
commit a19cd20
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 18 deletions.
diff --git a/api/app/models/metrics/drift_dto.py b/api/app/models/metrics/drift_dto.py
@@ -4,6 +4,7 @@
 from pydantic import BaseModel, ConfigDict
 from pydantic.alias_generators import to_camel
 
+from app.models.inferred_schema_dto import FieldType
 from app.models.job_status import JobStatus
 
 
@@ -23,6 +24,7 @@ class FeatureDriftCalculation(BaseModel):
 
 class FeatureMetrics(BaseModel):
     feature_name: str
+    field_type: FieldType
     drift_calc: FeatureDriftCalculation
 
     model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

diff --git a/api/tests/commons/db_mock.py b/api/tests/commons/db_mock.py
@@ -475,14 +475,17 @@ def get_sample_current_dataset(
     'featureMetrics': [
         {
             'featureName': 'gender',
+            'fieldType': 'categorical',
             'driftCalc': {'type': 'CHI2', 'value': 0.87, 'hasDrift': True},
         },
         {
             'featureName': 'city',
+            'fieldType': 'categorical',
             'driftCalc': {'type': 'CHI2', 'value': 0.12, 'hasDrift': False},
         },
         {
             'featureName': 'age',
+            'fieldType': 'numerical',
             'driftCalc': {'type': 'KS', 'value': 0.92, 'hasDrift': True},
         },
     ]

diff --git a/init-data/init_db.sql b/init-data/init_db.sql
diff --git a/init-data/mirror_files.sh b/init-data/mirror_files.sh
@@ -7,12 +7,12 @@
 /usr/bin/mc mb myminio/test-bucket
 
 # Copy files from the project root to the correct paths in the MinIO bucket
-/usr/bin/mc cp /data/binary_classification/reference-income.csv myminio/test-bucket/e02f2aed-cd29-4703-8faf-2dcab9fc668e/reference/ee43cc3d-d71c-43f2-be0b-55da73d509c5/
-/usr/bin/mc cp /data/binary_classification/current1-income.csv myminio/test-bucket/e02f2aed-cd29-4703-8faf-2dcab9fc668e/current/63b173f8-6150-41f5-8a6b-a1e08a3a7c12/
-/usr/bin/mc cp /data/multiclass-classification/3_classes_reference.csv myminio/test-bucket/756204a0-a6ba-44c2-b287-ca828e2b2441/reference/dd84c147-f6aa-4584-a2dc-3ca4cd00917b/
-/usr/bin/mc cp /data/multiclass-classification/3_classes_current1.csv myminio/test-bucket/756204a0-a6ba-44c2-b287-ca828e2b2441/current/658f7c87-b557-4611-a29e-9ae0ae42ded7/
-/usr/bin/mc cp /data/regression/regression_abalone_reference.csv myminio/test-bucket/6c48c7e1-442b-4c48-a218-95d07f40a805/reference/12fbe35a-3530-4e26-b53b-ee12dc47a874/
-/usr/bin/mc cp /data/regression/regression_abalone_current1.csv myminio/test-bucket/6c48c7e1-442b-4c48-a218-95d07f40a805/current/03616777-0b1e-47ed-b1ad-273dc51d467d/
+/usr/bin/mc cp /data/binary_classification/reference-income.csv myminio/test-bucket/e02f2aed-cd29-4703-8faf-2dcab9fc668e/reference/db15546e-16a3-4482-8317-323d66dbe409/
+/usr/bin/mc cp /data/binary_classification/current1-income.csv myminio/test-bucket/e02f2aed-cd29-4703-8faf-2dcab9fc668e/current/b73334fb-aac0-400c-9103-fe894e13bd95/
+/usr/bin/mc cp /data/multiclass-classification/3_classes_reference.csv myminio/test-bucket/756204a0-a6ba-44c2-b287-ca828e2b2441/reference/d8cb36a9-2f8b-4e45-bab0-aeaa259a9f46/
+/usr/bin/mc cp /data/multiclass-classification/3_classes_current1.csv myminio/test-bucket/756204a0-a6ba-44c2-b287-ca828e2b2441/current/dda74c14-91de-4ae8-a432-3a2f8ccdd153/
+/usr/bin/mc cp /data/regression/regression_abalone_reference.csv myminio/test-bucket/6c48c7e1-442b-4c48-a218-95d07f40a805/reference/6ae1390e-656a-45aa-acd4-6f6b38e6acf4/
+/usr/bin/mc cp /data/regression/regression_abalone_current1.csv myminio/test-bucket/6c48c7e1-442b-4c48-a218-95d07f40a805/current/edb6ed6c-6e2d-44fa-8a60-40540aec9c03/
 
 # Exit the script
 exit 0
diff --git a/sdk/radicalbit_platform_sdk/models/dataset_drift.py b/sdk/radicalbit_platform_sdk/models/dataset_drift.py
@@ -4,6 +4,8 @@
 from pydantic import BaseModel, ConfigDict
 from pydantic.alias_generators import to_camel
 
+from radicalbit_platform_sdk.models.field_type import FieldType
+
 
 class DriftAlgorithm(str, Enum):
     KS = 'KS'
@@ -21,6 +23,7 @@ class FeatureDriftCalculation(BaseModel):
 
 class FeatureDrift(BaseModel):
     feature_name: str
+    field_type: FieldType
     drift_calc: FeatureDriftCalculation
 
     model_config = ConfigDict(populate_by_name=True, alias_generator=to_camel)

diff --git a/sdk/tests/apis/model_current_dataset_test.py b/sdk/tests/apis/model_current_dataset_test.py
@@ -14,6 +14,7 @@
     CurrentRegressionModelQuality,
     Drift,
     DriftAlgorithm,
+    FieldType,
     JobStatus,
     ModelType,
     RegressionDataQuality,
@@ -166,14 +167,17 @@ def test_drift_ok(self):
                         "featureMetrics": [
                             {
                                 "featureName": "gender",
+                                "fieldType": "categorical",
                                 "driftCalc": {"type": "CHI2", "value": 0.87, "hasDrift": true}
                             },
                             {
                                 "featureName": "city",
+                                "fieldType": "categorical",
                                 "driftCalc": {"type": "CHI2", "value": 0.12, "hasDrift": false}
                             },
                             {
                                 "featureName": "age",
+                                "fieldType": "numerical",
                                 "driftCalc": {"type": "KS", "value": 0.92, "hasDrift": true}
                             }
                         ]
@@ -187,10 +191,12 @@ def test_drift_ok(self):
 
         assert len(drift.feature_metrics) == 3
         assert drift.feature_metrics[1].feature_name == 'city'
+        assert drift.feature_metrics[1].field_type == FieldType.categorical
         assert drift.feature_metrics[1].drift_calc.type == DriftAlgorithm.CHI2
         assert drift.feature_metrics[1].drift_calc.value == 0.12
         assert drift.feature_metrics[1].drift_calc.has_drift is False
         assert drift.feature_metrics[2].feature_name == 'age'
+        assert drift.feature_metrics[2].field_type == FieldType.numerical
         assert drift.feature_metrics[2].drift_calc.type == DriftAlgorithm.KS
         assert drift.feature_metrics[2].drift_calc.value == 0.92
         assert drift.feature_metrics[2].drift_calc.has_drift is True