Full_preprocessing_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merging individual Treatment Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "pressor = pd.read_csv(\"files/Pressor.csv\")\n",
    "ventilation = pd.read_csv(\"files/ventilation.csv\")\n",
    "dialysis = pd.read_csv(\"files/Dialysis.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Vasopressor: Unite into one activity from the first to the last timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\adams\\anaconda3\\lib\\site-packages\\pandas\\core\\generic.py:3936: PerformanceWarning: dropping on a non-lexsorted multi-index without a level parameter may impact performance.\n",
      "  obj = obj._drop_axis(labels, axis, level=level, errors=errors)\n"
     ]
    }
   ],
   "source": [
    "agg_timestamps = {'starttime': ['first'],'endtime': ['last']}\n",
    "pressor_new = pressor.groupby(['stay_id','linkorderid'], as_index=False).agg(agg_timestamps)\n",
    "pressor_new = pressor_new.drop(columns=[\"linkorderid\"], axis = 1)\n",
    "pressor_new = pressor_new.droplevel(level=1, axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Vasopressor: Generalize all vasopressor under the name pressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "pressor_new[\"activity\"] = \"PRESSOR\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dialysis: Unite ongoing Dialysis into one event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = dialysis.to_numpy()\n",
    "activated = False\n",
    "activated_tr = \"\"\n",
    "activated_stay = 0\n",
    "activated_line = 0\n",
    "last_activated_time = \"\"\n",
    "\n",
    "lines_to_drop = []\n",
    "end_timestamps = ['0' for i in range(0,len(arr))]\n",
    "\n",
    "last_stay = 0\n",
    "last_stat = 0\n",
    "#clean activatedness\n",
    "for line in range(0,len(arr)):\n",
    "    curr_stay = arr[line][0]\n",
    "    curr_stat = arr[line][3]\n",
    "    if curr_stat == 1:\n",
    "        if last_stat == 0:\n",
    "            activated = True\n",
    "            activated_tr = arr[line][4]\n",
    "            activated_stay = curr_stay\n",
    "            activated_line = line\n",
    "            last_activated_time = arr[line][1]\n",
    "        if last_stat == 1:\n",
    "            #check if same treatment\n",
    "            if curr_stay == activated_stay and activated_tr == arr[line][4]:\n",
    "                last_activated_time = arr[line][1]\n",
    "                lines_to_drop.append(line)\n",
    "            else: \n",
    "                end_timestamps[activated_line] = last_activated_time\n",
    "                #store new treatment information\n",
    "                activated = True\n",
    "                activated_tr = arr[line][4]\n",
    "                activated_stay = curr_stay\n",
    "                activated_line = line\n",
    "                last_activated_time = arr[line][1]\n",
    "    else:\n",
    "        activated = False\n",
    "        lines_to_drop.append(line)\n",
    "        if last_stat == 1:\n",
    "            #check if same treatment\n",
    "            if curr_stay == activated_stay and activated_tr == arr[line][4]:\n",
    "                end_timestamps[activated_line] = arr[line][1]\n",
    "            else: \n",
    "                end_timestamps[activated_line] = last_activated_time\n",
    "    last_stat = curr_stat\n",
    "    last_stay = curr_stay\n",
    "if activated:\n",
    "    end_timestamps[activated_line] = last_activated_time\n",
    "dialysis[\"endtime\"] = end_timestamps\n",
    "dialysis = dialysis.drop(lines_to_drop,axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "dialysis[\"activity\"] = \"DIALYSIS\"\n",
    "dialysis = dialysis.drop([\"dialysis_present\",\"dialysis_active\",\"dialysis_type\"],axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ventilation: Rename Activity and Drop the status info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "ventilation[\"activity\"] = \"VENTILATION\"\n",
    "ventilation = ventilation.drop([\"ventilation_status\"],axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge to an event log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "log = pd.concat([dialysis, ventilation, pressor_new])\n",
    "log[\"starttime\"] = pd.to_datetime(log[\"starttime\"])\n",
    "log[\"endtime\"] = pd.to_datetime(log[\"endtime\"])\n",
    "log.sort_values(by='starttime', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Join treatments of the same kind to one single treatment with start and end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "log = log.groupby(['stay_id','activity'], as_index=False).agg(agg_timestamps)\n",
    "log = log.droplevel(level=1, axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add Patient Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "pat = pd.read_csv(\"files/patient_data.csv\") ####Add the control variables and comorbidities here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "log = log.join(pat.set_index('stay_id'),on=\"stay_id\", how=\"left\",rsuffix=\"_r\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "log[\"language\"] = log[\"language\"].replace(\"?\",\"Non-native speaker\")\n",
    "log[\"language\"] = log[\"language\"].replace(\"ENGLISH\",\"Native english speaker\")\n",
    "log[\"language\"].value_counts()\n",
    "log = log[log['language'].notna()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add admit and discharge event for each patient and sofa score for each event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "sofa = pd.read_csv(\"files/sofa_scores_update.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timedelta\n",
    "log_arr_new = []\n",
    "log_arr = [x for _,x in log.groupby(\"stay_id\")]\n",
    "for df in log_arr:\n",
    "    first_row = df.iloc[0].copy()\n",
    "    last_row = df.iloc[0].copy()\n",
    "    sofa_scores = sofa[sofa[\"stay_id\"]==first_row[\"stay_id\"]].sort_values(\"starttime\")\n",
    "    sofa_scores[\"starttime\"] = pd.to_datetime(sofa_scores[\"starttime\"])\n",
    "    sofa_scores[\"endtime\"] = pd.to_datetime(sofa_scores[\"endtime\"])\n",
    "    df[\"starttime\"] = pd.to_datetime(df[\"starttime\"])\n",
    "    df[\"endtime\"] = pd.to_datetime(df[\"endtime\"])\n",
    "    \n",
    "    first_row[\"activity\"] = \"ADMIT\"\n",
    "    last_row[\"activity\"] = \"DISCHARGE\"\n",
    "    first_row[\"starttime\"] = first_row[\"admittime\"]\n",
    "    first_row[\"endtime\"] = first_row[\"admittime\"]\n",
    "    last_row[\"starttime\"] = last_row[\"dischtime\"]\n",
    "    last_row[\"endtime\"] = first_row[\"dischtime\"]\n",
    "    first_row[\"SOFA\"] = sofa_scores.iloc[0][\"sofa_24hours\"]\n",
    "    last_row[\"SOFA\"] = sofa_scores.iloc[-1][\"sofa_24hours\"]\n",
    "    sofa_score_arr = sofa_scores.to_numpy()\n",
    "    new_df = pd.DataFrame({})\n",
    "    for i_row, row in df.iterrows():\n",
    "        found = False\n",
    "        for i in range(0,len(sofa_score_arr)):\n",
    "            #some sofas are recorded after the first treatment, they will be assigned with the first recorded sofa score\n",
    "            if sofa_score_arr[i][1] <= row[\"starttime\"] and sofa_score_arr[i][2]>= row[\"starttime\"]:\n",
    "                row[\"SOFA\"] = sofa_score_arr[i][3]\n",
    "                found = True\n",
    "                break\n",
    "        if not found:\n",
    "            row[\"SOFA\"] = sofa_score_arr[0][3]\n",
    "        new_df = pd.concat([new_df,row.to_frame().T])  \n",
    "    log_arr_new.append(pd.concat([new_df,last_row.to_frame().T,first_row.to_frame().T]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "log = pd.concat(log_arr_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "log = log.sort_values(by=[\"stay_id\",\"starttime\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Write the event log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "log.to_csv(\"files/event_log.csv\",index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}