Skip to content

Commit

Permalink
Fix bug with null replication metrics when row is all null (#706)
Browse files Browse the repository at this point in the history
* Fix bug when row is all null

* Improve test

* Remove unnecessary type

* Improve efficiency
  • Loading branch information
tonywu315 authored Nov 8, 2022
1 parent 64ccd8e commit 98e2c45
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 1 deletion.
6 changes: 5 additions & 1 deletion dataprofiler/profilers/profile_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -2287,7 +2287,11 @@ def _update_null_replication_metrics(self, clean_samples: Dict) -> None:
# Partition data based on whether target column value is null or not
# Calculate sum, mean of each partition without including current column
# in calculation
sum_null = data.iloc[null_indices, data.columns != col_id].sum().to_numpy()
sum_null = (
data.loc[data.index.intersection(null_indices), data.columns != col_id]
.sum()
.to_numpy()
)

# Add old sum_null if exists
if col_id in self._null_replication_metrics:
Expand Down
15 changes: 15 additions & 0 deletions dataprofiler/tests/profilers/test_profile_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -2081,6 +2081,21 @@ def test_null_replication_metrics_calculation(self):
np.testing.assert_array_almost_equal([[np.nan], [18]], column["class_sum"])
np.testing.assert_array_almost_equal([[np.nan], [9]], column["class_mean"])

# Test with all null in a row
data_4 = pd.DataFrame(
[[10, 20], [9999999, 9999999], [30, 9999999], [9999999, 9999999]]
)

profiler = dp.StructuredProfiler(data_4, options=profile_options)
report = profiler.report()

self.assertTrue("null_replication_metrics" in report["data_stats"][0])
column = report["data_stats"][0]["null_replication_metrics"]

np.testing.assert_array_almost_equal([0.5, 0.5], column["class_prior"])
np.testing.assert_array_almost_equal([[20], [0]], column["class_sum"])
np.testing.assert_array_almost_equal([[10], [0]], column["class_mean"])

def test_column_level_invalid_values(self):
data = pd.DataFrame([[1, 1], [9999999, 2], [3, 3]])

Expand Down

0 comments on commit 98e2c45

Please sign in to comment.