preprocessing/preprocess_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This notebook reads in the discretised input data and then preprocesses the model features\n",
    "# Firstly, values deemed excessively high/low are capped\n",
    "# Relevant binary features and normally/log-normally features are standardised accordingly\n",
    "# Training and test sets are split - 70% train, 10% validation, 20% test\n",
    "# Resulting datasets are saved to file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from pandas import DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "disc_inp_data = pd.read_csv(\"../data/discretised_input_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0      224552\n",
      " 100     15583\n",
      "-100      2315\n",
      "Name: reward, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# add rewards - sparsely for now; reward function shaping comes in a separate script\n",
    "disc_inp_data['reward'] = 0\n",
    "for i in disc_inp_data.index:\n",
    "    if i == 0:\n",
    "        continue\n",
    "    else:\n",
    "        if disc_inp_data.loc[i, 'icustayid'] != disc_inp_data.loc[i-1, 'icustayid']:\n",
    "            if disc_inp_data.loc[i-1, 'died_in_hosp'] == 1:\n",
    "                disc_inp_data.loc[i-1,'reward'] = -100\n",
    "            elif disc_inp_data.loc[i-1, 'died_in_hosp'] == 0:\n",
    "                disc_inp_data.loc[i-1,'reward'] = 100\n",
    "            else:\n",
    "                print \"error in row\", i-1\n",
    "if disc_inp_data.loc[len(disc_inp_data)-1, 'died_in_hosp'] == 1:\n",
    "    disc_inp_data.loc[len(disc_inp_data)-1, 'reward'] = -100\n",
    "elif disc_inp_data.loc[len(disc_inp_data)-1, 'died_in_hosp'] == 0:\n",
    "     disc_inp_data.loc[len(disc_inp_data)-1, 'reward'] = 100\n",
    "print disc_inp_data['reward'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# now split into train/validation/test sets\n",
    "import random\n",
    "unique_ids = disc_inp_data['icustayid'].unique()\n",
    "random.shuffle(unique_ids)\n",
    "train_sample = 0.7\n",
    "val_sample = 0.1\n",
    "test_sample = 0.2\n",
    "train_num = int(len(unique_ids) * 0.7)\n",
    "val_num = int(len(unique_ids)*0.1) + train_num\n",
    "train_ids = unique_ids[:train_num]\n",
    "val_ids = unique_ids[train_num:val_num]\n",
    "test_ids = unique_ids[val_num:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set = DataFrame()\n",
    "train_set = disc_inp_data.loc[disc_inp_data['icustayid'].isin(train_ids)]\n",
    "\n",
    "val_set = DataFrame()\n",
    "val_set = disc_inp_data.loc[disc_inp_data['icustayid'].isin(val_ids)]\n",
    "\n",
    "test_set = DataFrame()\n",
    "test_set = disc_inp_data.loc[disc_inp_data['icustayid'].isin(test_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n",
      "/usr/local/lib/python2.7/dist-packages/pandas/core/generic.py:5088: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._update_inplace(new_data)\n",
      "/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py:2882: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # This is added back by InteractiveShellApp.init_path()\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if sys.path[0] == '':\n"
     ]
    }
   ],
   "source": [
    "# cap values in train and test\n",
    "caps = pd.read_csv(\"capping_values.csv\")\n",
    "for i in caps.index:\n",
    "    param = caps.loc[i,'parameter'][1:-1]\n",
    "    maxval = caps.loc[i,'limsup']\n",
    "    minval = caps.loc[i,'liminf']\n",
    "    train_set[param][train_set[param] >= maxval] = maxval\n",
    "    train_set[param][train_set[param] <= minval] = minval\n",
    "    val_set[param][val_set[param] >= maxval] = maxval\n",
    "    val_set[param][val_set[param] <= minval] = minval\n",
    "    test_set[param][test_set[param] >= maxval] = maxval\n",
    "    test_set[param][test_set[param] <= minval] = minval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "binary_fields = ['gender','mechvent','re_admission']\n",
    "norm_fields= ['age','Weight_kg','GCS','HR','SysBP','MeanBP','DiaBP','RR','Temp_C','FiO2_1',\n",
    "    'Potassium','Sodium','Chloride','Glucose','Magnesium','Calcium',\n",
    "    'Hb','WBC_count','Platelets_count','PTT','PT','Arterial_pH','paO2','paCO2',\n",
    "    'Arterial_BE','HCO3','Arterial_lactate','SOFA','SIRS','Shock_Index',\n",
    "    'PaO2_FiO2','cumulated_balance_tev', 'elixhauser', 'Albumin', u'CO2_mEqL', 'Ionised_Ca']\n",
    "log_fields = ['max_dose_vaso','SpO2','BUN','Creatinine','SGOT','SGPT','Total_bili','INR',\n",
    "              'input_total_tev','input_4hourly_tev','output_total','output_4hourly', 'bloc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/pandas/core/frame.py:2450: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self[k1] = value[k2]\n"
     ]
    }
   ],
   "source": [
    "# normalise binary fields\n",
    "train_set[binary_fields] = train_set[binary_fields] - 0.5 \n",
    "val_set[binary_fields] = val_set[binary_fields] - 0.5 \n",
    "test_set[binary_fields] = test_set[binary_fields] - 0.5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n"
     ]
    }
   ],
   "source": [
    "# normal distn fields\n",
    "for item in norm_fields:\n",
    "    av = train_set[item].mean()\n",
    "    std = train_set[item].std()\n",
    "    train_set[item] = (train_set[item] - av) / std\n",
    "    val_set[item] = (val_set[item] - av) / std\n",
    "    test_set[item] = (test_set[item] - av) / std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    }
   ],
   "source": [
    "# log normal fields\n",
    "train_set[log_fields] = np.log(0.1 + train_set[log_fields])\n",
    "val_set[log_fields] = np.log(0.1 + val_set[log_fields])\n",
    "test_set[log_fields] = np.log(0.1 + test_set[log_fields])\n",
    "for item in log_fields:\n",
    "    av = train_set[item].mean()\n",
    "    std = train_set[item].std()\n",
    "    train_set[item] = (train_set[item] - av) / std\n",
    "    val_set[item] = (val_set[item] - av) / std\n",
    "    test_set[item] = (test_set[item] - av) / std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bloc</th>\n",
       "      <th>icustayid</th>\n",
       "      <th>charttime</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>elixhauser</th>\n",
       "      <th>re_admission</th>\n",
       "      <th>died_in_hosp</th>\n",
       "      <th>mortality_90d</th>\n",
       "      <th>Weight_kg</th>\n",
       "      <th>...</th>\n",
       "      <th>median_dose_vaso</th>\n",
       "      <th>max_dose_vaso</th>\n",
       "      <th>input_total_tev</th>\n",
       "      <th>input_4hourly_tev</th>\n",
       "      <th>output_total</th>\n",
       "      <th>output_4hourly</th>\n",
       "      <th>cumulated_balance_tev</th>\n",
       "      <th>vaso_input</th>\n",
       "      <th>iv_input</th>\n",
       "      <th>reward</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-2.311573</td>\n",
       "      <td>3</td>\n",
       "      <td>7245052800</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.973552</td>\n",
       "      <td>-1.803826</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.218124</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.378832</td>\n",
       "      <td>0.139523</td>\n",
       "      <td>1.257985</td>\n",
       "      <td>-0.376479</td>\n",
       "      <td>0.422937</td>\n",
       "      <td>0.097512</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.472264</td>\n",
       "      <td>3</td>\n",
       "      <td>7245067200</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.973552</td>\n",
       "      <td>-1.803826</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.218124</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.378832</td>\n",
       "      <td>0.324790</td>\n",
       "      <td>1.243446</td>\n",
       "      <td>-0.037480</td>\n",
       "      <td>0.645655</td>\n",
       "      <td>0.326743</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.966747</td>\n",
       "      <td>3</td>\n",
       "      <td>7245081600</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.973552</td>\n",
       "      <td>-1.803826</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.218124</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.378832</td>\n",
       "      <td>0.332546</td>\n",
       "      <td>0.374560</td>\n",
       "      <td>0.327893</td>\n",
       "      <td>1.033948</td>\n",
       "      <td>0.102300</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.603852</td>\n",
       "      <td>3</td>\n",
       "      <td>7245096000</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.973552</td>\n",
       "      <td>-1.803826</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.218124</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.378832</td>\n",
       "      <td>0.336913</td>\n",
       "      <td>0.209693</td>\n",
       "      <td>0.400830</td>\n",
       "      <td>0.734782</td>\n",
       "      <td>0.018508</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.320563</td>\n",
       "      <td>3</td>\n",
       "      <td>7245110400</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-0.973552</td>\n",
       "      <td>-1.803826</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.218124</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.378832</td>\n",
       "      <td>0.341234</td>\n",
       "      <td>0.209693</td>\n",
       "      <td>0.442817</td>\n",
       "      <td>0.619696</td>\n",
       "      <td>-0.036855</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       bloc  icustayid   charttime  gender       age  elixhauser  \\\n",
       "0 -2.311573          3  7245052800    -0.5 -0.973552   -1.803826   \n",
       "1 -1.472264          3  7245067200    -0.5 -0.973552   -1.803826   \n",
       "2 -0.966747          3  7245081600    -0.5 -0.973552   -1.803826   \n",
       "3 -0.603852          3  7245096000    -0.5 -0.973552   -1.803826   \n",
       "4 -0.320563          3  7245110400    -0.5 -0.973552   -1.803826   \n",
       "\n",
       "   re_admission  died_in_hosp  mortality_90d  Weight_kg   ...    \\\n",
       "0          -0.5             0              1  -0.218124   ...     \n",
       "1          -0.5             0              1  -0.218124   ...     \n",
       "2          -0.5             0              1  -0.218124   ...     \n",
       "3          -0.5             0              1  -0.218124   ...     \n",
       "4          -0.5             0              1  -0.218124   ...     \n",
       "\n",
       "   median_dose_vaso  max_dose_vaso  input_total_tev  input_4hourly_tev  \\\n",
       "0               0.0      -0.378832         0.139523           1.257985   \n",
       "1               0.0      -0.378832         0.324790           1.243446   \n",
       "2               0.0      -0.378832         0.332546           0.374560   \n",
       "3               0.0      -0.378832         0.336913           0.209693   \n",
       "4               0.0      -0.378832         0.341234           0.209693   \n",
       "\n",
       "   output_total  output_4hourly  cumulated_balance_tev  vaso_input  iv_input  \\\n",
       "0     -0.376479        0.422937               0.097512         0.0       4.0   \n",
       "1     -0.037480        0.645655               0.326743         0.0       4.0   \n",
       "2      0.327893        1.033948               0.102300         0.0       2.0   \n",
       "3      0.400830        0.734782               0.018508         0.0       2.0   \n",
       "4      0.442817        0.619696              -0.036855         0.0       2.0   \n",
       "\n",
       "   reward  \n",
       "0       0  \n",
       "1       0  \n",
       "2       0  \n",
       "3       0  \n",
       "4       0  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set.to_csv('../data/rl_train_set_unscaled.csv',index = False)\n",
    "val_set.to_csv('../data/rl_val_set_unscaled.csv', index = False)\n",
    "test_set.to_csv('../data/rl_test_set_unscaled.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    }
   ],
   "source": [
    "# scale features to [0,1] in train set, similar in val and test\n",
    "import copy\n",
    "scalable_fields = copy.deepcopy(binary_fields)\n",
    "scalable_fields.extend(norm_fields)\n",
    "scalable_fields.extend(log_fields)\n",
    "for col in scalable_fields:\n",
    "    minimum = min(train_set[col])\n",
    "    maximum = max(train_set[col])\n",
    "    train_set[col] = (train_set[col] - minimum)/(maximum-minimum)\n",
    "    val_set[col] = (val_set[col] - minimum)/(maximum-minimum)\n",
    "    test_set[col] = (test_set[col] - minimum)/(maximum-minimum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bloc</th>\n",
       "      <th>icustayid</th>\n",
       "      <th>charttime</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "      <th>elixhauser</th>\n",
       "      <th>re_admission</th>\n",
       "      <th>died_in_hosp</th>\n",
       "      <th>mortality_90d</th>\n",
       "      <th>Weight_kg</th>\n",
       "      <th>...</th>\n",
       "      <th>median_dose_vaso</th>\n",
       "      <th>max_dose_vaso</th>\n",
       "      <th>input_total_tev</th>\n",
       "      <th>input_4hourly_tev</th>\n",
       "      <th>output_total</th>\n",
       "      <th>output_4hourly</th>\n",
       "      <th>cumulated_balance_tev</th>\n",
       "      <th>vaso_input</th>\n",
       "      <th>iv_input</th>\n",
       "      <th>reward</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>7245052800</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.412568</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.262712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.797351</td>\n",
       "      <td>0.939195</td>\n",
       "      <td>0.589916</td>\n",
       "      <td>0.750908</td>\n",
       "      <td>0.554500</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.222560</td>\n",
       "      <td>3</td>\n",
       "      <td>7245067200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.412568</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.262712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.831780</td>\n",
       "      <td>0.934543</td>\n",
       "      <td>0.674384</td>\n",
       "      <td>0.819589</td>\n",
       "      <td>0.580033</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.356608</td>\n",
       "      <td>3</td>\n",
       "      <td>7245081600</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.412568</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.262712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.833222</td>\n",
       "      <td>0.656575</td>\n",
       "      <td>0.765423</td>\n",
       "      <td>0.939329</td>\n",
       "      <td>0.555033</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.452837</td>\n",
       "      <td>3</td>\n",
       "      <td>7245096000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.412568</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.262712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.834033</td>\n",
       "      <td>0.603831</td>\n",
       "      <td>0.783597</td>\n",
       "      <td>0.847073</td>\n",
       "      <td>0.545700</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.527957</td>\n",
       "      <td>3</td>\n",
       "      <td>7245110400</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.412568</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.262712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.834836</td>\n",
       "      <td>0.603831</td>\n",
       "      <td>0.794059</td>\n",
       "      <td>0.811583</td>\n",
       "      <td>0.539533</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       bloc  icustayid   charttime  gender       age  elixhauser  \\\n",
       "0  0.000000          3  7245052800     0.0  0.412568         0.0   \n",
       "1  0.222560          3  7245067200     0.0  0.412568         0.0   \n",
       "2  0.356608          3  7245081600     0.0  0.412568         0.0   \n",
       "3  0.452837          3  7245096000     0.0  0.412568         0.0   \n",
       "4  0.527957          3  7245110400     0.0  0.412568         0.0   \n",
       "\n",
       "   re_admission  died_in_hosp  mortality_90d  Weight_kg   ...    \\\n",
       "0           0.0             0              1   0.262712   ...     \n",
       "1           0.0             0              1   0.262712   ...     \n",
       "2           0.0             0              1   0.262712   ...     \n",
       "3           0.0             0              1   0.262712   ...     \n",
       "4           0.0             0              1   0.262712   ...     \n",
       "\n",
       "   median_dose_vaso  max_dose_vaso  input_total_tev  input_4hourly_tev  \\\n",
       "0               0.0            0.0         0.797351           0.939195   \n",
       "1               0.0            0.0         0.831780           0.934543   \n",
       "2               0.0            0.0         0.833222           0.656575   \n",
       "3               0.0            0.0         0.834033           0.603831   \n",
       "4               0.0            0.0         0.834836           0.603831   \n",
       "\n",
       "   output_total  output_4hourly  cumulated_balance_tev  vaso_input  iv_input  \\\n",
       "0      0.589916        0.750908               0.554500         0.0       4.0   \n",
       "1      0.674384        0.819589               0.580033         0.0       4.0   \n",
       "2      0.765423        0.939329               0.555033         0.0       2.0   \n",
       "3      0.783597        0.847073               0.545700         0.0       2.0   \n",
       "4      0.794059        0.811583               0.539533         0.0       2.0   \n",
       "\n",
       "   reward  \n",
       "0       0  \n",
       "1       0  \n",
       "2       0  \n",
       "3       0  \n",
       "4       0  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_set.to_csv('../data/rl_train_set_scaled.csv',index = False)\n",
    "val_set.to_csv('../data/rl_val_set_scaled.csv', index = False)\n",
    "test_set.to_csv('../data/rl_test_set_scaled.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}