feat: percent empty

app/schema.graphql

@@ -5,6 +5,7 @@ type Cluster {
 
 enum DataQualityMetric {
   cardinality
+  percentEmpty
 }
 
 type Dataset {
@@ -21,7 +22,7 @@ type Dimension implements Node {
   name: String!
   type: DimensionType!
   dataType: DimensionDataType!
-  dataQualityMetric(metric: DataQualityMetric!): Int
+  dataQualityMetric(metric: DataQualityMetric!): Float
 }
 
 type DimensionConnection {

app/src/components/model/ModelSchemaTable.tsx

@@ -26,6 +26,7 @@ export function ModelSchemaTable(props: ModelSchemaTableProps) {
                 type
                 dataType
                 cardinality: dataQualityMetric(metric: cardinality)
+                percentEmpty: dataQualityMetric(metric: percentEmpty)
               }
             }
           }
@@ -64,6 +65,10 @@ export function ModelSchemaTable(props: ModelSchemaTableProps) {
         Header: "Cardinality",
         accessor: "cardinality",
       },
+      {
+        Header: "Percent Empty",
+        accessor: "percentEmpty",
+      },
     ];
     return cols;
   }, []);

src/phoenix/metrics/cardinality.py

@@ -1,7 +1,3 @@
-"""
-Cardinality metrics
-"""
-
 import concurrent.futures as cf
 from typing import Dict, List, Optional
 

src/phoenix/metrics/percent_empty.py

@@ -0,0 +1,14 @@
+from typing import Dict, List, Optional
+
+from pandas import DataFrame
+
+
+def percent_empty(dataframe: DataFrame, column_names: List[str]) -> Dict[str, Optional[float]]:
+    """
+    Returns a map of the dataframe column names to the percent of empty entries
+    for each row.
+    """
+    num_records = dataframe.shape[0]
+    if num_records == 0:
+        return {col: None for col in column_names}
+    return dict(dataframe[column_names].isnull().sum() / num_records)

src/phoenix/server/api/loaders.py

@@ -6,16 +6,19 @@
 from phoenix.core import DimensionDataType
 from phoenix.core.model import Model
 from phoenix.metrics.cardinality import cardinality
+from phoenix.metrics.percent_empty import percent_empty
 
 
 @dataclass
 class Loaders:
     cardinality: DataLoader[str, Optional[int]]
+    percent_empty: DataLoader[str, Optional[float]]
 
 
 def create_loaders(model: Model) -> Loaders:
     return Loaders(
         cardinality=_get_cardinality_dataloader(model=model),
+        percent_empty=_get_percent_empty_dataloader(model=model),
     )
 
 
@@ -39,3 +42,13 @@ async def _cardinality_load_function(column_names: List[str]) -> List[Optional[i
         return [column_name_to_cardinality[col] for col in column_names]
 
     return DataLoader(load_fn=_cardinality_load_function)
+
+
+def _get_percent_empty_dataloader(model: Model) -> DataLoader[str, Optional[float]]:
+    async def _percent_empty_load_function(column_names: List[str]) -> List[Optional[float]]:
+        column_name_to_percent_empty = percent_empty(
+            dataframe=model.primary_dataset.dataframe, column_names=column_names
+        )
+        return [column_name_to_percent_empty[col] for col in column_names]
+
+    return DataLoader(load_fn=_percent_empty_load_function)

src/phoenix/server/api/types/DataQualityMetric.py

@@ -6,3 +6,4 @@
 @strawberry.enum
 class DataQualityMetric(Enum):
     cardinality = "cardinality"
+    percentEmpty = "percentEmpty"

src/phoenix/server/api/types/Dimension.py

@@ -21,9 +21,12 @@ class Dimension(Node):
     @strawberry.field
     async def dataQualityMetric(
         self, metric: DataQualityMetric, info: Info[Context, None]
-    ) -> Optional[int]:
+    ) -> Optional[float]:
+        dimension_name = self.name
         if metric is DataQualityMetric.cardinality:
-            return await info.context.loaders.cardinality.load(self.name)
+            return await info.context.loaders.cardinality.load(dimension_name)
+        elif metric is DataQualityMetric.percentEmpty:
+            return await info.context.loaders.percent_empty.load(dimension_name)
         raise NotImplementedError(f"Metric {metric} is not implemented.")
 
 

tests/metrics/drift/test_percent_empty.py

@@ -0,0 +1,41 @@
+from numpy.testing import assert_array_almost_equal
+from pandas import DataFrame
+
+from phoenix.metrics.percent_empty import percent_empty
+
+
+def test_percent_empty_returns_correct_percents_including_for_empty_and_full_columns():
+    dataframe = DataFrame(
+        {
+            "col0": [None, None, None],
+            "col1": [1.0, None, None],
+            "col2": ["string-entry", None, "another-string-entry"],
+            "col3": [0.1, 0.2, 0.3],
+        }
+    )
+    expected_column_name_to_percent_empty = {
+        "col0": 1.0,
+        "col1": 2 / 3,
+        "col2": 1 / 3,
+        "col3": 0.0,
+    }
+    expected_column_names = ["col0", "col1", "col2", "col3"]
+    column_name_to_percent_empty = percent_empty(
+        dataframe=dataframe, column_names=["col0", "col1", "col2", "col3"]
+    )
+    assert expected_column_names == sorted(column_name_to_percent_empty.keys())
+    assert_array_almost_equal(
+        [column_name_to_percent_empty[col] for col in expected_column_names],
+        [expected_column_name_to_percent_empty[col] for col in expected_column_names],
+    )
+
+
+def test_percent_empty_returns_only_input_columns():
+    dataframe = DataFrame(
+        {
+            "col0": [1, 2, None],
+            "col1": [1.0, None, None],
+        }
+    )
+    column_name_to_percent_empty = percent_empty(dataframe=dataframe, column_names=["col0"])
+    assert ["col0"] == list(column_name_to_percent_empty.keys())