Modelling_part_2_with_pool_difference_in_shape_v1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from sys import platform\n",
    "if platform  == 'win32':\n",
    "    sys.argv=[\"_\", r\"P:\\rmahajan14\\capstone_data\\data\", r\"P:\\rmahajan14\\capstone_data\\cache\"]\n",
    "if platform == 'linux' or platform == 'linux2':\n",
    "    def full_path(p):\n",
    "        return os.path.abspath(os.path.expanduser(p))\n",
    "    sys.argv=[\"_\", r\"/mnt/d/DiDi_Ride_Data/data\", r\"/mnt/d/DiDi_Ride_Data/Ride_Data_Cache\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from loader1 import *\n",
    "from utils import *\n",
    "from common import *\n",
    "from experiment2 import *\n",
    "# from sys import platform\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Basic Coding "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "spatial_type = 'radial'\n",
    "combine_pool = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "orders\n",
      "Decay Calculation\n",
      "No Decay\n",
      "(5909529, 6)\n",
      "(5909529, 4)\n",
      "(1181102, 4)\n",
      "(1181102, 2)\n",
      "(1181102, 3)\n",
      "(945550, 6)\n",
      "(1181102, 6)\n",
      "Dumping to ../Ride_Data_Cache/active_times_pool_True.msgpack\n",
      "Decay Calculation done in 156.77280688285828\n",
      "1e\n",
      "../Ride_Data_Cache/features_orders.msgpack exists\n",
      "(1181102, 12)\n",
      "Features created in 0.8946182727813721\n",
      "1f\n",
      "../Ride_Data_Cache/radial_spatial_df.msgpack exists\n",
      "spatial\n",
      "Spatial Calculation done in 3.5409820079803467\n"
     ]
    }
   ],
   "source": [
    "df_final_no_decay, target_df_no_decay = get_final_df_reg(use_cache=True, decay='No Decay', \n",
    "                                                         spatial_type=spatial_type, combine_pool=combine_pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/features_orders.msgpack exists\n"
     ]
    }
   ],
   "source": [
    "df = create_features(use_cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1181102, 12)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "a\n",
      "Loading from ../Ride_Data_Cache/groupby1.msgpack\n",
      "Loading from ../Ride_Data_Cache/groupby2.msgpack\n",
      "1.2721872329711914\n",
      "c\n",
      "d\n",
      "Dumping to ../Ride_Data_Cache/features_orders.msgpack\n"
     ]
    }
   ],
   "source": [
    "df2 = create_features(use_cache=False, save_file=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1181102, 12)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "orders\n",
      "Decay Calculation\n",
      "Old Decay\n",
      "Creating the exponential decay\n",
      "Dumping to ../Ride_Data_Cache/idle_times_old_pool_True.msgpack\n",
      "Decay Calculation done in 45.78079605102539\n",
      "1e\n",
      "../Ride_Data_Cache/features_orders.msgpack exists\n",
      "(1181102, 12)\n",
      "Features created in 0.7628128528594971\n",
      "1f\n",
      "../Ride_Data_Cache/radial_spatial_df.msgpack exists\n",
      "spatial\n",
      "Spatial Calculation done in 3.481710910797119\n"
     ]
    }
   ],
   "source": [
    "df_final_old_decay, target_df_old_decay = get_final_df_reg(use_cache=True, decay='Old Decay',\n",
    "                                                           spatial_type=spatial_type,combine_pool=combine_pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Using the function create_modified_active_time_through_decay2 \n",
    "# instead of create_modified_active_time_through_decay\n",
    "df_final_new_decay1, target_df_new_decay1 = get_final_df_reg(use_cache=True, decay='New Decay', mult_factor=1, spatial_type=spatial_type, combine_pool=combine_pool)\n",
    "df_final_new_decay10, target_df_new_decay10 = get_final_df_reg(use_cache=True, decay='New Decay', mult_factor=10, spatial_type=spatial_type, combine_pool=combine_pool)\n",
    "df_final_new_decay100, target_df_new_decay100 = get_final_df_reg(use_cache=True, decay='New Decay', mult_factor=100, spatial_type=spatial_type, combine_pool=combine_pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_df = df_final_new_decay100[df_final_new_decay100['num_pool_rides'] == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_df_new_decay10.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final_new_decay100.sort_values('num_pool_rides', ascending=False).head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start = '2016-11-01'\n",
    "# end = '2016-11-30'\n",
    "# orders = merge_order_df(start=start, end=end, use_cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# orders[['ride_start_timestamp', 'ride_stop_timestamp']][orders.driver_id == '3d5914545a97b002450e7769d7e51df3'].sort_values('ride_start_timestamp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_target_df = target_df_new_decay100[target_df_new_decay100.driver_id.isin(list(new_df.index))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_target_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_target_df.shape, new_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_target_df[new_target_df.target > 1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n"
     ]
    }
   ],
   "source": [
    "orders = merge_order_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>ride_start_timestamp</th>\n",
       "      <th>ride_stop_timestamp</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>ride_duration</th>\n",
       "      <th>driver_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>114178</td>\n",
       "      <td>8c27811a5adc40de896a7b915d9cc6ef</td>\n",
       "      <td>2016-11-05 18:20:07</td>\n",
       "      <td>2016-11-05 18:40:15</td>\n",
       "      <td>104.043000</td>\n",
       "      <td>30.703900</td>\n",
       "      <td>104.071400</td>\n",
       "      <td>30.710100</td>\n",
       "      <td>20.133333</td>\n",
       "      <td>0131839f8e956896fac162ada060dc40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>67726</td>\n",
       "      <td>53385eed47907edcfb3d936b1f82429f</td>\n",
       "      <td>2016-11-05 18:43:21</td>\n",
       "      <td>2016-11-05 19:33:06</td>\n",
       "      <td>104.066498</td>\n",
       "      <td>30.709419</td>\n",
       "      <td>104.040586</td>\n",
       "      <td>30.666127</td>\n",
       "      <td>49.750000</td>\n",
       "      <td>0131839f8e956896fac162ada060dc40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>136001</td>\n",
       "      <td>a70cb0fc8abfb4e9428dafb864e193d7</td>\n",
       "      <td>2016-11-05 18:50:06</td>\n",
       "      <td>2016-11-05 19:07:14</td>\n",
       "      <td>104.064935</td>\n",
       "      <td>30.709150</td>\n",
       "      <td>104.043810</td>\n",
       "      <td>30.702740</td>\n",
       "      <td>17.133333</td>\n",
       "      <td>0131839f8e956896fac162ada060dc40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73528</td>\n",
       "      <td>5a3c2ea59c81d11837c82d1fe27054f9</td>\n",
       "      <td>2016-11-05 19:20:32</td>\n",
       "      <td>2016-11-05 19:48:27</td>\n",
       "      <td>104.045550</td>\n",
       "      <td>30.691100</td>\n",
       "      <td>104.055720</td>\n",
       "      <td>30.646970</td>\n",
       "      <td>27.916667</td>\n",
       "      <td>0131839f8e956896fac162ada060dc40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                order_id ride_start_timestamp  \\\n",
       "114178  8c27811a5adc40de896a7b915d9cc6ef  2016-11-05 18:20:07   \n",
       "67726   53385eed47907edcfb3d936b1f82429f  2016-11-05 18:43:21   \n",
       "136001  a70cb0fc8abfb4e9428dafb864e193d7  2016-11-05 18:50:06   \n",
       "73528   5a3c2ea59c81d11837c82d1fe27054f9  2016-11-05 19:20:32   \n",
       "\n",
       "       ride_stop_timestamp  pickup_longitude  pickup_latitude  \\\n",
       "114178 2016-11-05 18:40:15        104.043000        30.703900   \n",
       "67726  2016-11-05 19:33:06        104.066498        30.709419   \n",
       "136001 2016-11-05 19:07:14        104.064935        30.709150   \n",
       "73528  2016-11-05 19:48:27        104.045550        30.691100   \n",
       "\n",
       "        dropoff_longitude  dropoff_latitude  ride_duration  \\\n",
       "114178         104.071400         30.710100      20.133333   \n",
       "67726          104.040586         30.666127      49.750000   \n",
       "136001         104.043810         30.702740      17.133333   \n",
       "73528          104.055720         30.646970      27.916667   \n",
       "\n",
       "                               driver_id  \n",
       "114178  0131839f8e956896fac162ada060dc40  \n",
       "67726   0131839f8e956896fac162ada060dc40  \n",
       "136001  0131839f8e956896fac162ada060dc40  \n",
       "73528   0131839f8e956896fac162ada060dc40  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orders[orders.driver_id == '0131839f8e956896fac162ada060dc40']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "orders\n",
      "Decay Calculation\n",
      "Survival\n",
      "Creating the Survival Functions\n",
      "Dumping to ../Ride_Data_Cache/survival_probability_df_pool_True.msgpack\n",
      "Decay Calculation done in 46.896864891052246\n",
      "1e\n",
      "../Ride_Data_Cache/features_orders.msgpack exists\n",
      "Features created in 0.778083086013794\n",
      "1f\n",
      "../Ride_Data_Cache/radial_spatial_df.msgpack exists\n",
      "spatial\n",
      "Spatial Calculation done in 3.2769579887390137\n"
     ]
    }
   ],
   "source": [
    "# Using the function create_modified_active_time_through_decay2 \n",
    "# instead of create_modified_active_time_through_decay\n",
    "df_final_new_decay_surv, target_df_new_decay_surv = get_final_df_reg(use_cache=True, decay='Survival', \n",
    "                                                                     spatial_type=spatial_type, combine_pool=combine_pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1181071, 112), (1181071, 5))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final_new_decay_surv.shape, target_df_new_decay_surv.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(186, 5)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_df_new_decay_surv[target_df_new_decay_surv['inactive_time'] < -1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# target = get_surv_prob(orders, save_file=False, use_cache=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# orders = merge_order_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# orders[orders.driver_id == '00032bc2d22da93ecfaa7b25a9a6f445'].sort_values('ride_start_timestamp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [df_final_no_decay, df_final_old_decay, df_final_new_decay1, \n",
    "           df_final_new_decay10, df_final_new_decay100, df_final_new_decay_surv]\n",
    "target_df_list = [target_df_no_decay, target_df_old_decay, target_df_new_decay1, \n",
    "                  target_df_new_decay10, target_df_new_decay100, target_df_new_decay_surv]\n",
    "name_list = ['no_decay', 'old_decay', 'new_decay1', 'new_decay10', 'new_decay100', 'survival']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [df_final_new_decay_surv]\n",
    "target_df_list = [target_df_new_decay_surv]\n",
    "name_list = ['survival']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "survival 0.0\n"
     ]
    }
   ],
   "source": [
    "for name, target in zip(name_list, target_df_list):\n",
    "    print(name, 100*target[target.target > 1].shape[0]/target.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " target_df_new_decay100[target_df_new_decay100.target > 1].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now to debug why greater than 0%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start = '2016-11-01'\n",
    "# end = '2016-11-30'\n",
    "# orders = merge_order_df(start=start, end=end, use_cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tmp = orders[['driver_id', 'ride_start_timestamp', 'ride_stop_timestamp', 'ride_duration']][orders.driver_id == '0001860739024029fa3da2cad0ed4de2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tmp['ride_duration'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tmp.sort_values('ride_start_timestamp').head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# actual_ride_duration = 53 + 32 + 31 + 58 + 48 + 27 + 22\n",
    "# actual_ride_duration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Printing results for 0001860739024029fa3da2cad0ed4de2\n",
    "\n",
    "# target_df = create_modified_active_time_through_decay2(orders, use_cache=False, save_file=False, mult_factor=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that ride duration is summed 3 times which have overlapping periods due to which it becomes greater than 1. Also as you can see above, the actual inactive time in row 33721 should be 28 but we make it 95 which cuts 60 minutes from the drives active time further increasing the error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# target_df[target_df.ride_duration > target_df.modified_active_time].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# val = ['0001860739024029fa3da2cad0ed4de2', '000211a97ba2b768ef941593bf1ae1a5', '0009cc512604d27cc3e819ab6b954fca', '3d5914545a97b002450e7769d7e51df3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# order_filter = orders[orders.driver_id.isin(val)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# order_filter.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now to see those who are active for more than 20 hours "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# target_df.sort_values('modified_active_time', ascending=False).head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Printing results for d18747ec3c039917014c1b397b528d97\n",
    "\n",
    "# target_df_new = create_modified_active_time_through_decay2(orders, use_cache=False, save_file=False, mult_factor=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "# orders[['ride_start_timestamp', 'ride_stop_timestamp', 'ride_duration']][orders.driver_id == 'd18747ec3c039917014c1b397b528d97'].sort_values('ride_start_timestamp')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In essence we are always saying a driver will be active between rides for 10 minutes at least "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modelling Code "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have till now coded the following models \n",
    "\n",
    "1. Linear Regression \n",
    "2. Linear Regression with L1 Regularization\n",
    "3. Linear Regression with L2 Regularization\n",
    "4. Random Forest Regressor "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we split the data in a ratio of 75:25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV, LassoCV, ElasticNet\n",
    "from sklearn.ensemble import RandomForestRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(df, target_df, name, seed=0):\n",
    "    # regression forest\n",
    "    print(f'building regression model for {name}')\n",
    "    X = df    \n",
    "    xtrain, xtest, ytrain, ytest = train_test_split(X, target_df['target'], random_state=seed)\n",
    "\n",
    "    sc = StandardScaler()\n",
    "\n",
    "    xtrain_sc = sc.fit_transform(xtrain)\n",
    "\n",
    "    rr = RandomForestRegressor(random_state=seed)\n",
    "    rr.fit(xtrain_sc, ytrain)\n",
    "\n",
    "    xtest_sc = sc.transform(xtest)\n",
    "    \n",
    "    print(rr.score(xtest_sc, ytest))\n",
    "    \n",
    "    return rr, rr.score(xtest_sc, ytest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final_new_decay_surv.shape, target_df_new_decay_surv.shape\n",
    "# df_final_no_decay.shape, target_df_no_decay.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [df_final_no_decay, df_final_old_decay, df_final_new_decay1, \n",
    "           df_final_new_decay10, df_final_new_decay100, df_final_new_decay_surv]\n",
    "target_df_list = [target_df_no_decay, target_df_old_decay, target_df_new_decay1, \n",
    "                  target_df_new_decay10, target_df_new_decay100, target_df_new_decay_surv]\n",
    "name_list = ['no_decay', 'old_decay', 'new_decay1', 'new_decay10', 'new_decay100', 'survival']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_list = [new_df]\n",
    "# target_df_list = [new_target_df]\n",
    "# name_list = ['No_Pool']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building regression model for survival\n",
      "0.5774645701649963\n"
     ]
    }
   ],
   "source": [
    "result = {}\n",
    "model = {}\n",
    "for df, target_df, name in zip(df_list, target_df_list, name_list):\n",
    "    model[name], result[name] = build_model(df, target_df, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_list = [df_final_new_decay100]\n",
    "# target_df_list = [target_df_new_decay100]\n",
    "# name_list = ['Decay100']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for df, target_df, name in zip(df_list, target_df_list, name_list):\n",
    "#     model[name], result[name] = build_model(df, target_df, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list(new_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest = model['new_decay100']\n",
    "X = df_final_new_decay100\n",
    "importances = forest.feature_importances_\n",
    "std = np.std([tree.feature_importances_ for tree in forest.estimators_],\n",
    "             axis=0)\n",
    "indices = np.argsort(importances)[::-1]\n",
    "\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(X.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], importances[indices[f]]))\n",
    "\n",
    "# Plot the feature importances of the forest\n",
    "plt.figure()\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(X.shape[1]), importances[indices],\n",
    "       color=\"r\", yerr=std[indices], align=\"center\")\n",
    "plt.xticks(range(X.shape[1]), indices)\n",
    "plt.xlim([-1, X.shape[1]])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest = model['survival']\n",
    "X = df_final_new_decay_surv\n",
    "importances = forest.feature_importances_\n",
    "std = np.std([tree.feature_importances_ for tree in forest.estimators_],\n",
    "             axis=0)\n",
    "indices = np.argsort(importances)[::-1]\n",
    "\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(X.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], importances[indices[f]]))\n",
    "\n",
    "# Plot the feature importances of the forest\n",
    "plt.figure()\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(X.shape[1]), importances[indices],\n",
    "       color=\"r\", yerr=std[indices], align=\"center\")\n",
    "plt.xticks(range(X.shape[1]), indices)\n",
    "plt.xlim([-1, X.shape[1]])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# col = [x.shape[1] for x in df_list]\n",
    "# col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set(df_list[1].columns).difference(df_list[0].columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final_old_decay.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final_new_decay10.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}