Add min_impute and nan warning

healthcareai/common/transformers.py

@@ -2,6 +2,7 @@
 
 This module contains transformers for preprocessing data. Most operate on DataFrames and are named appropriately.
 """
+import warnings
 import numpy as np
 import pandas as pd
 
@@ -15,16 +16,26 @@ class DataFrameImputer(TransformerMixin):
  """
  Impute missing values in a dataframe.
 
- Columns of dtype object or category (assumed categorical) are imputed with the mode (most frequent value in column).
+ Columns of dtype object or category (assumed categorical) are imputed 
+ with the mode (most frequent value in column).
 
  Columns of other types (assumed continuous) are imputed with mean of column.
  """
 
- def __init__(self, impute=True, verbose=True):
+ def __init__(self, impute=True, max_impute=.5, verbose=True):
+ """
+ Initiates the DataFrameImputer with imputation flag and threshold.
+
+ Args:
+ impute: Flag on whether to impute
+ max_impute: Maximum proportion of values to impute, will warn
+ the user if the threshold is passed
+ """
  self.impute = impute
  self.object_columns = None
  self.fill = None
  self.verbose = verbose
+ self.max_impute = max_impute
 
  def fit(self, X, y=None):
  # Return if not imputing
@@ -54,7 +65,15 @@ def transform(self, X, y=None):
  # Return if not imputing
  if self.impute is False:
  return X
-
+
+ #Warn users if %nan is too high
+ for c in X:
+ pct_impute = X[c].isnull().sum() / len(X)
+ if pct_impute > self.max_impute:
+ warnings.warn("'{0}' column is missing {1:.2f}% of data. Imputed "
+ "values may be invalid.".format(c, pct_impute*100),
+ RuntimeWarning)
+
  result = X.fillna(self.fill)
 
  for i in self.object_columns: