Modelling_part_2_with_pool.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from sys import platform\n",
    "if platform  == 'win32':\n",
    "    sys.argv=[\"_\", r\"P:\\rmahajan14\\capstone_data\\data\", r\"P:\\rmahajan14\\capstone_data\\cache\"]\n",
    "if platform == 'linux' or platform == 'linux2':\n",
    "    def full_path(p):\n",
    "        return os.path.abspath(os.path.expanduser(p))\n",
    "    sys.argv=[\"_\", r\"/mnt/d/DiDi_Ride_Data/data\", r\"/mnt/d/DiDi_Ride_Data/Ride_Data_Cache\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from loader1 import *\n",
    "from utils import *\n",
    "from common import *\n",
    "from experiment2 import *\n",
    "# from sys import platform\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Basic Coding "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "spatial_type = 'radial'\n",
    "combine_pool = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "orders\n",
      "Decay Calculation\n",
      "No Decay\n",
      "Dumping to ../Ride_Data_Cache/active_times_pool_True.msgpack\n",
      "Decay Calculation done in 117.86984705924988\n",
      "1e\n",
      "../Ride_Data_Cache/features_orders.msgpack exists\n",
      "Features created in 0.7458748817443848\n",
      "1f\n",
      "../Ride_Data_Cache/radial_spatial_df.msgpack exists\n",
      "spatial\n",
      "Spatial Calculation done in 3.358247995376587\n",
      "> /Users/ashwin/Documents/Courses/Capstone/capstone_didi/experiment2.py(298)get_final_df_reg()\n",
      "-> if sum(df_final.index != target_df['driver_id']) > 0:\n",
      "(Pdb) df_final.head(4)\n",
      "                                  num_total_rides  num_pool_rides  \\\n",
      "driver_id                                                           \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                3             0.0   \n",
      "00002724a19c5f6a54ae8d60a378997e                1             0.0   \n",
      "000042288ba964aa04d14869bd56efb6                5             0.0   \n",
      "000049d47b79747ad9927ba1bf5f4126               11             1.0   \n",
      "\n",
      "                                  % of pool rides  \\\n",
      "driver_id                                           \n",
      "0000131d486b69eb77ab6e9e7cca9f4c         0.000000   \n",
      "00002724a19c5f6a54ae8d60a378997e         0.000000   \n",
      "000042288ba964aa04d14869bd56efb6         0.000000   \n",
      "000049d47b79747ad9927ba1bf5f4126         0.090909   \n",
      "\n",
      "                                 (0 days 00:00:44.999999, 0 days 10:36:30]_count  \\\n",
      "driver_id                                                                          \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                               0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                               1   \n",
      "000042288ba964aa04d14869bd56efb6                                               0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                        0.363636   \n",
      "\n",
      "                                 (0 days 10:36:30, 0 days 14:43:06]_count  \\\n",
      "driver_id                                                                   \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                        1   \n",
      "00002724a19c5f6a54ae8d60a378997e                                        0   \n",
      "000042288ba964aa04d14869bd56efb6                                        0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                0.0909091   \n",
      "\n",
      "                                 (0 days 14:43:06, 0 days 18:45:25]_count  \\\n",
      "driver_id                                                                   \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                        0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                        0   \n",
      "000042288ba964aa04d14869bd56efb6                                        0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                 0.545455   \n",
      "\n",
      "                                 (0 days 18:45:25, 0 days 23:59:59]_count  \\\n",
      "driver_id                                                                   \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                        0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                        0   \n",
      "000042288ba964aa04d14869bd56efb6                                        1   \n",
      "000049d47b79747ad9927ba1bf5f4126                                        0   \n",
      "\n",
      "                                 (0 days 00:00:44.999999, 0 days 10:36:30]_sum  \\\n",
      "driver_id                                                                        \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                             0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                             1   \n",
      "000042288ba964aa04d14869bd56efb6                                             0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                      0.347145   \n",
      "\n",
      "                                 (0 days 10:36:30, 0 days 14:43:06]_sum  \\\n",
      "driver_id                                                                 \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                      1   \n",
      "00002724a19c5f6a54ae8d60a378997e                                      0   \n",
      "000042288ba964aa04d14869bd56efb6                                      0   \n",
      "000049d47b79747ad9927ba1bf5f4126                               0.083383   \n",
      "\n",
      "                                 (0 days 14:43:06, 0 days 18:45:25]_sum  ...  \\\n",
      "driver_id                                                                ...   \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                      0  ...   \n",
      "00002724a19c5f6a54ae8d60a378997e                                      0  ...   \n",
      "000042288ba964aa04d14869bd56efb6                                      0  ...   \n",
      "000049d47b79747ad9927ba1bf5f4126                               0.569472  ...   \n",
      "\n",
      "                                 (9509.75, 78107.176](-180.001, -99.688]  \\\n",
      "driver_id                                                                  \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                     0.0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                     0.0   \n",
      "000042288ba964aa04d14869bd56efb6                                     0.0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                     0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](-38.414, -8.521]  \\\n",
      "driver_id                                                                 \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                    0.0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                    0.0   \n",
      "000042288ba964aa04d14869bd56efb6                                    0.0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                    0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](-8.521, 14.323]  \\\n",
      "driver_id                                                                \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                   0.0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                   0.0   \n",
      "000042288ba964aa04d14869bd56efb6                                   0.0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                   0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](-99.688, -38.414]  \\\n",
      "driver_id                                                                  \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                     0.0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                     0.0   \n",
      "000042288ba964aa04d14869bd56efb6                                     0.0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                     0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](111.762, 137.56]  \\\n",
      "driver_id                                                                 \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                    0.0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                    0.0   \n",
      "000042288ba964aa04d14869bd56efb6                                    0.0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                    0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](137.56, 180.0]  \\\n",
      "driver_id                                                               \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                  0.0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                  0.0   \n",
      "000042288ba964aa04d14869bd56efb6                                  0.0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                  0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](14.323, 38.147]  \\\n",
      "driver_id                                                                \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                   0.0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                   0.0   \n",
      "000042288ba964aa04d14869bd56efb6                                   0.0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                   0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](38.147, 60.214]  \\\n",
      "driver_id                                                                \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                   0.0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                   0.0   \n",
      "000042288ba964aa04d14869bd56efb6                                   0.0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                   0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](60.214, 88.383]  \\\n",
      "driver_id                                                                \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                   0.0   \n",
      "00002724a19c5f6a54ae8d60a378997e                                   0.0   \n",
      "000042288ba964aa04d14869bd56efb6                                   0.0   \n",
      "000049d47b79747ad9927ba1bf5f4126                                   0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](88.383, 111.762]  \n",
      "driver_id                                                                \n",
      "0000131d486b69eb77ab6e9e7cca9f4c                                    0.0  \n",
      "00002724a19c5f6a54ae8d60a378997e                                    0.0  \n",
      "000042288ba964aa04d14869bd56efb6                                    0.0  \n",
      "000049d47b79747ad9927ba1bf5f4126                                    0.0  \n",
      "\n",
      "[4 rows x 111 columns]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Pdb) sum(df_final.index != target_df['driver_id'])\n",
      "*** ValueError: Lengths must match to compare\n",
      "(Pdb) df_final.shape\n",
      "(1181102, 111)\n",
      "(Pdb) target_df.shape\n",
      "(1181071, 5)\n",
      "(Pdb) df_temp = set(df_final.index).difference(set(target_df['driver_id']))\n",
      "(Pdb) df_temp\n",
      "{'c774aa09ce03e8d47cf337da58302543', 'a89a994295da8a90d6d21638240d7376', '48eb8bc15129557a7ed8733abc794d2b', 'e89f7c4dda38938294588d206cf1da6d', '34c623dc540499c524a8d268cc5c9dea', '55977ce03f1117934403e1733a42fbde', '51a27187f29a7190918ac104abf3d522', 'cbbdb62f0ab1b5fdfff07839b6a8adc3', '30feebbe310386333e174e8e647e8db2', '44df981813d16d5a529e60f5eb706429', '800c0a4d1a97e6f7bbc91ad83226e606', 'bcab628f38691c3e6f415838cd6e88a1', 'bf877001e3942c79a25f38e99c4ea7ae', 'ad220cf9b3ffa351d7e3d5b6dd90e562', '650badb08e2c4acea61f8ac92972c5e5', '9bb8a09ee5c4592689950543b7639ff6', 'bbb8cee6c99f60845c1b038c47ba2bec', 'ec6f83cff0c5f510c39e78d7cb3524c4', '2897775527a20fabde934343f3b9f27d', 'ff9d0ef3ed475dae549a61be997e9686', 'e756994f50f85d028859e4a4e0ab059c', '7db705674497c2a8f2cbec000fce23b1', 'b11cc00ce1fbe6a4d7a3ef536a7c2c0c', '09756b369f95733345673061964c8eff', '2b5222e1c1dca69731079aa9aed0e859', 'a1f2adb44a62c6d49e96696834fecaee', '8093d7563bd0d091fc7d905a09b8afb5', 'b73d3cbc9848fcffcc04bdffd07273a3', '8c7ae94345e735c9760d8a7057d3c734', 'f1588c2172987b61f28a95f5f35464db', 'a08d6a5ec4e108c5d29d7891d1a4bf4b'}\n",
      "(Pdb) df_temp2 = df_final[df_final['driver_id'].isin(df_temp)])\n",
      "*** SyntaxError: invalid syntax\n",
      "(Pdb) df_temp2 = df_final[df_final['driver_id'].isin(df_temp)]]\n",
      "*** SyntaxError: invalid syntax\n",
      "(Pdb) df_temp2 = df_final[df_final['driver_id'].isin(df_temp)]\n",
      "*** KeyError: 'driver_id'\n",
      "(Pdb) df_temp2 = df_final[df_final.index.isin(df_temp)])\n",
      "*** SyntaxError: invalid syntax\n",
      "(Pdb) df_temp2 = df_final[df_final.index.isin(df_temp)]\n",
      "(Pdb) df_temp2.head(5)\n",
      "                                  num_total_rides  num_pool_rides  \\\n",
      "driver_id                                                           \n",
      "09756b369f95733345673061964c8eff                2             1.0   \n",
      "2897775527a20fabde934343f3b9f27d                2             1.0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                2             1.0   \n",
      "30feebbe310386333e174e8e647e8db2                3             2.0   \n",
      "34c623dc540499c524a8d268cc5c9dea                3             2.0   \n",
      "\n",
      "                                  % of pool rides  \\\n",
      "driver_id                                           \n",
      "09756b369f95733345673061964c8eff         0.500000   \n",
      "2897775527a20fabde934343f3b9f27d         0.500000   \n",
      "2b5222e1c1dca69731079aa9aed0e859         0.500000   \n",
      "30feebbe310386333e174e8e647e8db2         0.666667   \n",
      "34c623dc540499c524a8d268cc5c9dea         0.666667   \n",
      "\n",
      "                                 (0 days 00:00:44.999999, 0 days 10:36:30]_count  \\\n",
      "driver_id                                                                          \n",
      "09756b369f95733345673061964c8eff                                               0   \n",
      "2897775527a20fabde934343f3b9f27d                                               0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                               0   \n",
      "30feebbe310386333e174e8e647e8db2                                               0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                               0   \n",
      "\n",
      "                                 (0 days 10:36:30, 0 days 14:43:06]_count  \\\n",
      "driver_id                                                                   \n",
      "09756b369f95733345673061964c8eff                                        0   \n",
      "2897775527a20fabde934343f3b9f27d                                        0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                        1   \n",
      "30feebbe310386333e174e8e647e8db2                                        1   \n",
      "34c623dc540499c524a8d268cc5c9dea                                        0   \n",
      "\n",
      "                                 (0 days 14:43:06, 0 days 18:45:25]_count  \\\n",
      "driver_id                                                                   \n",
      "09756b369f95733345673061964c8eff                                        0   \n",
      "2897775527a20fabde934343f3b9f27d                                        0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                        0   \n",
      "30feebbe310386333e174e8e647e8db2                                        0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                        1   \n",
      "\n",
      "                                 (0 days 18:45:25, 0 days 23:59:59]_count  \\\n",
      "driver_id                                                                   \n",
      "09756b369f95733345673061964c8eff                                        1   \n",
      "2897775527a20fabde934343f3b9f27d                                        1   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                        0   \n",
      "30feebbe310386333e174e8e647e8db2                                        0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                        0   \n",
      "\n",
      "                                 (0 days 00:00:44.999999, 0 days 10:36:30]_sum  \\\n",
      "driver_id                                                                        \n",
      "09756b369f95733345673061964c8eff                                             0   \n",
      "2897775527a20fabde934343f3b9f27d                                             0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                             0   \n",
      "30feebbe310386333e174e8e647e8db2                                             0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                             0   \n",
      "\n",
      "                                 (0 days 10:36:30, 0 days 14:43:06]_sum  \\\n",
      "driver_id                                                                 \n",
      "09756b369f95733345673061964c8eff                                      0   \n",
      "2897775527a20fabde934343f3b9f27d                                      0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                      1   \n",
      "30feebbe310386333e174e8e647e8db2                                      1   \n",
      "34c623dc540499c524a8d268cc5c9dea                                      0   \n",
      "\n",
      "                                 (0 days 14:43:06, 0 days 18:45:25]_sum  ...  \\\n",
      "driver_id                                                                ...   \n",
      "09756b369f95733345673061964c8eff                                      0  ...   \n",
      "2897775527a20fabde934343f3b9f27d                                      0  ...   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                      0  ...   \n",
      "30feebbe310386333e174e8e647e8db2                                      0  ...   \n",
      "34c623dc540499c524a8d268cc5c9dea                                      1  ...   \n",
      "\n",
      "                                 (9509.75, 78107.176](-180.001, -99.688]  \\\n",
      "driver_id                                                                  \n",
      "09756b369f95733345673061964c8eff                                     0.0   \n",
      "2897775527a20fabde934343f3b9f27d                                     0.0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                     0.0   \n",
      "30feebbe310386333e174e8e647e8db2                                     0.0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                     0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](-38.414, -8.521]  \\\n",
      "driver_id                                                                 \n",
      "09756b369f95733345673061964c8eff                                    0.0   \n",
      "2897775527a20fabde934343f3b9f27d                                    0.0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                    0.0   \n",
      "30feebbe310386333e174e8e647e8db2                                    0.0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                    0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](-8.521, 14.323]  \\\n",
      "driver_id                                                                \n",
      "09756b369f95733345673061964c8eff                                   0.0   \n",
      "2897775527a20fabde934343f3b9f27d                                   0.0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                   0.0   \n",
      "30feebbe310386333e174e8e647e8db2                                   0.0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                   0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](-99.688, -38.414]  \\\n",
      "driver_id                                                                  \n",
      "09756b369f95733345673061964c8eff                                     0.0   \n",
      "2897775527a20fabde934343f3b9f27d                                     0.0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                     0.0   \n",
      "30feebbe310386333e174e8e647e8db2                                     0.0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                     0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](111.762, 137.56]  \\\n",
      "driver_id                                                                 \n",
      "09756b369f95733345673061964c8eff                                    0.0   \n",
      "2897775527a20fabde934343f3b9f27d                                    0.0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                    0.0   \n",
      "30feebbe310386333e174e8e647e8db2                                    0.0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                    0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](137.56, 180.0]  \\\n",
      "driver_id                                                               \n",
      "09756b369f95733345673061964c8eff                                  0.0   \n",
      "2897775527a20fabde934343f3b9f27d                                  0.0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                  0.0   \n",
      "30feebbe310386333e174e8e647e8db2                                  0.0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                  0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](14.323, 38.147]  \\\n",
      "driver_id                                                                \n",
      "09756b369f95733345673061964c8eff                                   0.0   \n",
      "2897775527a20fabde934343f3b9f27d                                   0.0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                   0.0   \n",
      "30feebbe310386333e174e8e647e8db2                                   0.0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                   0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](38.147, 60.214]  \\\n",
      "driver_id                                                                \n",
      "09756b369f95733345673061964c8eff                                   0.0   \n",
      "2897775527a20fabde934343f3b9f27d                                   0.0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                   0.0   \n",
      "30feebbe310386333e174e8e647e8db2                                   0.0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                   0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](60.214, 88.383]  \\\n",
      "driver_id                                                                \n",
      "09756b369f95733345673061964c8eff                                   0.0   \n",
      "2897775527a20fabde934343f3b9f27d                                   0.0   \n",
      "2b5222e1c1dca69731079aa9aed0e859                                   0.0   \n",
      "30feebbe310386333e174e8e647e8db2                                   0.0   \n",
      "34c623dc540499c524a8d268cc5c9dea                                   0.0   \n",
      "\n",
      "                                  (9509.75, 78107.176](88.383, 111.762]  \n",
      "driver_id                                                                \n",
      "09756b369f95733345673061964c8eff                                    0.0  \n",
      "2897775527a20fabde934343f3b9f27d                                    0.0  \n",
      "2b5222e1c1dca69731079aa9aed0e859                                    0.0  \n",
      "30feebbe310386333e174e8e647e8db2                                    0.0  \n",
      "34c623dc540499c524a8d268cc5c9dea                                    0.0  \n",
      "\n",
      "[5 rows x 111 columns]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Pdb) exit()\n"
     ]
    },
    {
     "ename": "BdbQuit",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mBdbQuit\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-0164014c6191>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m df_final_no_decay, target_df_no_decay = get_final_df_reg(use_cache=True, decay='No Decay', \n\u001b[0;32m----> 2\u001b[0;31m                                                          spatial_type=spatial_type, combine_pool=combine_pool)\n\u001b[0m",
      "\u001b[0;32m~/Documents/Courses/Capstone/capstone_didi/experiment2.py\u001b[0m in \u001b[0;36mget_final_df_reg\u001b[0;34m(use_cache, decay, mult_factor, add_idle_time, use_radial_features, spatial_type, combine_pool)\u001b[0m\n\u001b[1;32m    296\u001b[0m         \u001b[0mdf_final\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'driver_id'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m         \u001b[0mbreakpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 298\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_final\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mtarget_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'driver_id'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    299\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Index must match\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    300\u001b[0m             \u001b[0mbreakpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/Courses/Capstone/capstone_didi/experiment2.py\u001b[0m in \u001b[0;36mget_final_df_reg\u001b[0;34m(use_cache, decay, mult_factor, add_idle_time, use_radial_features, spatial_type, combine_pool)\u001b[0m\n\u001b[1;32m    296\u001b[0m         \u001b[0mdf_final\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'driver_id'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m         \u001b[0mbreakpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 298\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_final\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mtarget_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'driver_id'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    299\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Index must match\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    300\u001b[0m             \u001b[0mbreakpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/bdb.py\u001b[0m in \u001b[0;36mtrace_dispatch\u001b[0;34m(self, frame, event, arg)\u001b[0m\n\u001b[1;32m     86\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;31m# None\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mevent\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'line'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mevent\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'call'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/bdb.py\u001b[0m in \u001b[0;36mdispatch_line\u001b[0;34m(self, frame)\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_here\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbreak_here\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muser_line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquitting\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mBdbQuit\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    114\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrace_dispatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mBdbQuit\u001b[0m: "
     ]
    }
   ],
   "source": [
    "df_final_no_decay, target_df_no_decay = get_final_df_reg(use_cache=True, decay='No Decay', \n",
    "                                                         spatial_type=spatial_type, combine_pool=combine_pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "orders\n",
      "Decay Calculation\n",
      "Old Decay\n",
      "Creating the exponential decay\n",
      "Dumping to ../Ride_Data_Cache/idle_times_old_pool_True.msgpack\n",
      "Decay Calculation done in 51.68490028381348\n",
      "1e\n",
      "../Ride_Data_Cache/features_orders.msgpack exists\n",
      "Features created in 1.287153959274292\n",
      "1f\n",
      "../Ride_Data_Cache/radial_spatial_df.msgpack exists\n",
      "spatial\n",
      "Spatial Calculation done in 6.078614950180054\n"
     ]
    }
   ],
   "source": [
    "df_final_old_decay, target_df_old_decay = get_final_df_reg(use_cache=True, decay='Old Decay',\n",
    "                                                           spatial_type=spatial_type,combine_pool=combine_pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "orders\n",
      "Decay Calculation\n",
      "New Decay\n",
      "Creating the exponential decay\n",
      "Dumping to ../Ride_Data_Cache/idle_times_new_1_pool_True.msgpack\n",
      "Decay Calculation done in 49.97449278831482\n",
      "1e\n",
      "../Ride_Data_Cache/features_orders.msgpack exists\n",
      "Features created in 0.7741298675537109\n",
      "1f\n",
      "../Ride_Data_Cache/radial_spatial_df.msgpack exists\n",
      "spatial\n",
      "Spatial Calculation done in 4.297355890274048\n",
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "orders\n",
      "Decay Calculation\n",
      "New Decay\n",
      "Creating the exponential decay\n",
      "Dumping to ../Ride_Data_Cache/idle_times_new_10_pool_True.msgpack\n",
      "Decay Calculation done in 62.70924687385559\n",
      "1e\n",
      "../Ride_Data_Cache/features_orders.msgpack exists\n",
      "Features created in 1.0863888263702393\n",
      "1f\n",
      "../Ride_Data_Cache/radial_spatial_df.msgpack exists\n",
      "spatial\n",
      "Spatial Calculation done in 6.913671016693115\n",
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "orders\n",
      "Decay Calculation\n",
      "New Decay\n",
      "Creating the exponential decay\n",
      "Dumping to ../Ride_Data_Cache/idle_times_new_100_pool_True.msgpack\n",
      "Decay Calculation done in 62.21518921852112\n",
      "1e\n",
      "../Ride_Data_Cache/features_orders.msgpack exists\n",
      "Features created in 0.8512449264526367\n",
      "1f\n",
      "../Ride_Data_Cache/radial_spatial_df.msgpack exists\n",
      "spatial\n",
      "Spatial Calculation done in 4.625882148742676\n"
     ]
    }
   ],
   "source": [
    "# Using the function create_modified_active_time_through_decay2 \n",
    "# instead of create_modified_active_time_through_decay\n",
    "df_final_new_decay1, target_df_new_decay1 = get_final_df_reg(use_cache=True, decay='New Decay', mult_factor=1, spatial_type=spatial_type, combine_pool=combine_pool)\n",
    "df_final_new_decay10, target_df_new_decay10 = get_final_df_reg(use_cache=True, decay='New Decay', mult_factor=10, spatial_type=spatial_type, combine_pool=combine_pool)\n",
    "df_final_new_decay100, target_df_new_decay100 = get_final_df_reg(use_cache=True, decay='New Decay', mult_factor=100, spatial_type=spatial_type, combine_pool=combine_pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_df = df_final_new_decay100[df_final_new_decay100['num_pool_rides'] == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_df_new_decay10.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final_new_decay100.sort_values('num_pool_rides', ascending=False).head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start = '2016-11-01'\n",
    "# end = '2016-11-30'\n",
    "# orders = merge_order_df(start=start, end=end, use_cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# orders[['ride_start_timestamp', 'ride_stop_timestamp']][orders.driver_id == '3d5914545a97b002450e7769d7e51df3'].sort_values('ride_start_timestamp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_target_df = target_df_new_decay100[target_df_new_decay100.driver_id.isin(list(new_df.index))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_target_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_target_df.shape, new_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# new_target_df[new_target_df.target > 1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n"
     ]
    }
   ],
   "source": [
    "orders = merge_order_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>ride_start_timestamp</th>\n",
       "      <th>ride_stop_timestamp</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>ride_duration</th>\n",
       "      <th>driver_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>114178</td>\n",
       "      <td>8c27811a5adc40de896a7b915d9cc6ef</td>\n",
       "      <td>2016-11-05 18:20:07</td>\n",
       "      <td>2016-11-05 18:40:15</td>\n",
       "      <td>104.043000</td>\n",
       "      <td>30.703900</td>\n",
       "      <td>104.071400</td>\n",
       "      <td>30.710100</td>\n",
       "      <td>20.133333</td>\n",
       "      <td>0131839f8e956896fac162ada060dc40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>67726</td>\n",
       "      <td>53385eed47907edcfb3d936b1f82429f</td>\n",
       "      <td>2016-11-05 18:43:21</td>\n",
       "      <td>2016-11-05 19:33:06</td>\n",
       "      <td>104.066498</td>\n",
       "      <td>30.709419</td>\n",
       "      <td>104.040586</td>\n",
       "      <td>30.666127</td>\n",
       "      <td>49.750000</td>\n",
       "      <td>0131839f8e956896fac162ada060dc40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>136001</td>\n",
       "      <td>a70cb0fc8abfb4e9428dafb864e193d7</td>\n",
       "      <td>2016-11-05 18:50:06</td>\n",
       "      <td>2016-11-05 19:07:14</td>\n",
       "      <td>104.064935</td>\n",
       "      <td>30.709150</td>\n",
       "      <td>104.043810</td>\n",
       "      <td>30.702740</td>\n",
       "      <td>17.133333</td>\n",
       "      <td>0131839f8e956896fac162ada060dc40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73528</td>\n",
       "      <td>5a3c2ea59c81d11837c82d1fe27054f9</td>\n",
       "      <td>2016-11-05 19:20:32</td>\n",
       "      <td>2016-11-05 19:48:27</td>\n",
       "      <td>104.045550</td>\n",
       "      <td>30.691100</td>\n",
       "      <td>104.055720</td>\n",
       "      <td>30.646970</td>\n",
       "      <td>27.916667</td>\n",
       "      <td>0131839f8e956896fac162ada060dc40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                order_id ride_start_timestamp  \\\n",
       "114178  8c27811a5adc40de896a7b915d9cc6ef  2016-11-05 18:20:07   \n",
       "67726   53385eed47907edcfb3d936b1f82429f  2016-11-05 18:43:21   \n",
       "136001  a70cb0fc8abfb4e9428dafb864e193d7  2016-11-05 18:50:06   \n",
       "73528   5a3c2ea59c81d11837c82d1fe27054f9  2016-11-05 19:20:32   \n",
       "\n",
       "       ride_stop_timestamp  pickup_longitude  pickup_latitude  \\\n",
       "114178 2016-11-05 18:40:15        104.043000        30.703900   \n",
       "67726  2016-11-05 19:33:06        104.066498        30.709419   \n",
       "136001 2016-11-05 19:07:14        104.064935        30.709150   \n",
       "73528  2016-11-05 19:48:27        104.045550        30.691100   \n",
       "\n",
       "        dropoff_longitude  dropoff_latitude  ride_duration  \\\n",
       "114178         104.071400         30.710100      20.133333   \n",
       "67726          104.040586         30.666127      49.750000   \n",
       "136001         104.043810         30.702740      17.133333   \n",
       "73528          104.055720         30.646970      27.916667   \n",
       "\n",
       "                               driver_id  \n",
       "114178  0131839f8e956896fac162ada060dc40  \n",
       "67726   0131839f8e956896fac162ada060dc40  \n",
       "136001  0131839f8e956896fac162ada060dc40  \n",
       "73528   0131839f8e956896fac162ada060dc40  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orders[orders.driver_id == '0131839f8e956896fac162ada060dc40']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "orders\n",
      "Decay Calculation\n",
      "Survival\n",
      "Creating the Survival Functions\n",
      "Dumping to ../Ride_Data_Cache/survival_probability_df_pool_False.msgpack\n",
      "Decay Calculation done in 30.319456815719604\n",
      "1e\n",
      "../Ride_Data_Cache/features_orders.msgpack exists\n",
      "(1181102, 12)\n",
      "Features created in 0.7823548316955566\n",
      "1f\n",
      "../Ride_Data_Cache/radial_spatial_df.msgpack exists\n",
      "spatial\n",
      "Spatial Calculation done in 3.9500060081481934\n"
     ]
    }
   ],
   "source": [
    "# Using the function create_modified_active_time_through_decay2 \n",
    "# instead of create_modified_active_time_through_decay\n",
    "combine_pool=False\n",
    "df_final_new_decay_surv, target_df_new_decay_surv = get_final_df_reg(use_cache=True, decay='Survival', \n",
    "                                                                     spatial_type=spatial_type, combine_pool=combine_pool\n",
    "                                                                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../Ride_Data_Cache/merged_orders.msgpack exists\n",
      "orders\n",
      "Decay Calculation\n",
      "Survival\n",
      "Creating the Survival Functions\n",
      "Dumping to ../Ride_Data_Cache/survival_probability_df_pool_False.msgpack\n",
      "Decay Calculation done in 32.69728112220764\n",
      "1e\n",
      "../Ride_Data_Cache/features_orders.msgpack exists\n",
      "(1181102, 12)\n",
      "Features created in 1.548980951309204\n",
      "1f\n",
      "../Ride_Data_Cache/radial_spatial_df.msgpack exists\n",
      "spatial\n",
      "Spatial Calculation done in 7.512912034988403\n"
     ]
    }
   ],
   "source": [
    "combine_pool=False\n",
    "df_final_new_decay_surv, target_df_new_decay_surv = get_final_df_reg(use_cache=True, decay='Survival', \n",
    "                                                                     spatial_type=spatial_type, combine_pool=combine_pool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1181102, 112), (1181102, 5))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 00005adc52a78e1630cfb50e4bd0f89e\n",
    "\n",
    "df_final_new_decay_surv.shape, target_df_new_decay_surv.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>driver_id</th>\n",
       "      <th>ride_duration</th>\n",
       "      <th>survival_active_time</th>\n",
       "      <th>inactive_time</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0000131d486b69eb77ab6e9e7cca9f4c</td>\n",
       "      <td>75.583333</td>\n",
       "      <td>84.226130</td>\n",
       "      <td>20.823870</td>\n",
       "      <td>0.897386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>00002724a19c5f6a54ae8d60a378997e</td>\n",
       "      <td>34.800000</td>\n",
       "      <td>34.800000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>000042288ba964aa04d14869bd56efb6</td>\n",
       "      <td>73.083333</td>\n",
       "      <td>102.854286</td>\n",
       "      <td>81.129048</td>\n",
       "      <td>0.710552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>000049d47b79747ad9927ba1bf5f4126</td>\n",
       "      <td>195.883333</td>\n",
       "      <td>243.633187</td>\n",
       "      <td>371.400146</td>\n",
       "      <td>0.804009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>00005adc52a78e1630cfb50e4bd0f89e</td>\n",
       "      <td>67.150000</td>\n",
       "      <td>83.600163</td>\n",
       "      <td>34.449837</td>\n",
       "      <td>0.803228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>000079a54709c3ce262a21327bebc5a7</td>\n",
       "      <td>251.216667</td>\n",
       "      <td>270.428229</td>\n",
       "      <td>273.738438</td>\n",
       "      <td>0.928959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0000824380faac87cc64aab251a65efd</td>\n",
       "      <td>36.200000</td>\n",
       "      <td>36.200000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0000839aee24e6f8d7856b7d5a89d665</td>\n",
       "      <td>113.683333</td>\n",
       "      <td>134.144395</td>\n",
       "      <td>43.622272</td>\n",
       "      <td>0.847470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>000088d299916f2e44ef335f3752ae61</td>\n",
       "      <td>91.250000</td>\n",
       "      <td>106.959924</td>\n",
       "      <td>182.406742</td>\n",
       "      <td>0.853123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0000a3c4817080300cf3ef7d58faaad1</td>\n",
       "      <td>81.766667</td>\n",
       "      <td>99.546424</td>\n",
       "      <td>357.320243</td>\n",
       "      <td>0.821392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0000beca799d88e6810faf33df585dec</td>\n",
       "      <td>57.933333</td>\n",
       "      <td>59.587276</td>\n",
       "      <td>33.112724</td>\n",
       "      <td>0.972243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0000ce9bfa00bd78fc08e3747d1551d9</td>\n",
       "      <td>82.283333</td>\n",
       "      <td>88.160470</td>\n",
       "      <td>9.356196</td>\n",
       "      <td>0.933336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0000da1850824413bba84ada2269a89a</td>\n",
       "      <td>30.016667</td>\n",
       "      <td>30.016667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0000da877ccfd3045d8b7b524e5ea375</td>\n",
       "      <td>111.983333</td>\n",
       "      <td>123.867748</td>\n",
       "      <td>20.082252</td>\n",
       "      <td>0.904056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0000dbe2f9b7a09864e39eb56a415726</td>\n",
       "      <td>446.866667</td>\n",
       "      <td>510.910193</td>\n",
       "      <td>136.973140</td>\n",
       "      <td>0.874648</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           driver_id  ride_duration  survival_active_time  \\\n",
       "0   0000131d486b69eb77ab6e9e7cca9f4c      75.583333             84.226130   \n",
       "1   00002724a19c5f6a54ae8d60a378997e      34.800000             34.800000   \n",
       "2   000042288ba964aa04d14869bd56efb6      73.083333            102.854286   \n",
       "3   000049d47b79747ad9927ba1bf5f4126     195.883333            243.633187   \n",
       "4   00005adc52a78e1630cfb50e4bd0f89e      67.150000             83.600163   \n",
       "5   000079a54709c3ce262a21327bebc5a7     251.216667            270.428229   \n",
       "6   0000824380faac87cc64aab251a65efd      36.200000             36.200000   \n",
       "7   0000839aee24e6f8d7856b7d5a89d665     113.683333            134.144395   \n",
       "8   000088d299916f2e44ef335f3752ae61      91.250000            106.959924   \n",
       "9   0000a3c4817080300cf3ef7d58faaad1      81.766667             99.546424   \n",
       "10  0000beca799d88e6810faf33df585dec      57.933333             59.587276   \n",
       "11  0000ce9bfa00bd78fc08e3747d1551d9      82.283333             88.160470   \n",
       "12  0000da1850824413bba84ada2269a89a      30.016667             30.016667   \n",
       "13  0000da877ccfd3045d8b7b524e5ea375     111.983333            123.867748   \n",
       "14  0000dbe2f9b7a09864e39eb56a415726     446.866667            510.910193   \n",
       "\n",
       "    inactive_time    target  \n",
       "0       20.823870  0.897386  \n",
       "1        0.000000  1.000000  \n",
       "2       81.129048  0.710552  \n",
       "3      371.400146  0.804009  \n",
       "4       34.449837  0.803228  \n",
       "5      273.738438  0.928959  \n",
       "6        0.000000  1.000000  \n",
       "7       43.622272  0.847470  \n",
       "8      182.406742  0.853123  \n",
       "9      357.320243  0.821392  \n",
       "10      33.112724  0.972243  \n",
       "11       9.356196  0.933336  \n",
       "12       0.000000  1.000000  \n",
       "13      20.082252  0.904056  \n",
       "14     136.973140  0.874648  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# When combine true is True, results dont match with Akshat \n",
    "target_df_new_decay_surv.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>driver_id</th>\n",
       "      <th>ride_duration</th>\n",
       "      <th>survival_active_time</th>\n",
       "      <th>inactive_time</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0000131d486b69eb77ab6e9e7cca9f4c</td>\n",
       "      <td>75.583333</td>\n",
       "      <td>82.948175</td>\n",
       "      <td>22.101825</td>\n",
       "      <td>0.911212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>00002724a19c5f6a54ae8d60a378997e</td>\n",
       "      <td>34.800000</td>\n",
       "      <td>34.800000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>000042288ba964aa04d14869bd56efb6</td>\n",
       "      <td>73.083333</td>\n",
       "      <td>90.934133</td>\n",
       "      <td>93.049201</td>\n",
       "      <td>0.803695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>000049d47b79747ad9927ba1bf5f4126</td>\n",
       "      <td>195.883333</td>\n",
       "      <td>227.770052</td>\n",
       "      <td>387.263282</td>\n",
       "      <td>0.860005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>00005adc52a78e1630cfb50e4bd0f89e</td>\n",
       "      <td>67.150000</td>\n",
       "      <td>75.876154</td>\n",
       "      <td>42.173846</td>\n",
       "      <td>0.884995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>000079a54709c3ce262a21327bebc5a7</td>\n",
       "      <td>251.216667</td>\n",
       "      <td>275.603389</td>\n",
       "      <td>268.563278</td>\n",
       "      <td>0.911515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0000824380faac87cc64aab251a65efd</td>\n",
       "      <td>36.200000</td>\n",
       "      <td>36.200000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0000839aee24e6f8d7856b7d5a89d665</td>\n",
       "      <td>113.683333</td>\n",
       "      <td>124.892909</td>\n",
       "      <td>52.873758</td>\n",
       "      <td>0.910246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>000088d299916f2e44ef335f3752ae61</td>\n",
       "      <td>91.250000</td>\n",
       "      <td>103.286103</td>\n",
       "      <td>186.080564</td>\n",
       "      <td>0.883468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0000a3c4817080300cf3ef7d58faaad1</td>\n",
       "      <td>81.766667</td>\n",
       "      <td>105.369048</td>\n",
       "      <td>351.497619</td>\n",
       "      <td>0.776003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0000beca799d88e6810faf33df585dec</td>\n",
       "      <td>57.933333</td>\n",
       "      <td>61.766060</td>\n",
       "      <td>30.933940</td>\n",
       "      <td>0.937948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0000ce9bfa00bd78fc08e3747d1551d9</td>\n",
       "      <td>82.283333</td>\n",
       "      <td>89.695416</td>\n",
       "      <td>7.821250</td>\n",
       "      <td>0.917364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0000da1850824413bba84ada2269a89a</td>\n",
       "      <td>30.016667</td>\n",
       "      <td>30.016667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0000da877ccfd3045d8b7b524e5ea375</td>\n",
       "      <td>111.983333</td>\n",
       "      <td>129.300312</td>\n",
       "      <td>14.649688</td>\n",
       "      <td>0.866072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0000dbe2f9b7a09864e39eb56a415726</td>\n",
       "      <td>446.866667</td>\n",
       "      <td>499.270403</td>\n",
       "      <td>148.612930</td>\n",
       "      <td>0.895039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           driver_id  ride_duration  survival_active_time  \\\n",
       "0   0000131d486b69eb77ab6e9e7cca9f4c      75.583333             82.948175   \n",
       "1   00002724a19c5f6a54ae8d60a378997e      34.800000             34.800000   \n",
       "2   000042288ba964aa04d14869bd56efb6      73.083333             90.934133   \n",
       "3   000049d47b79747ad9927ba1bf5f4126     195.883333            227.770052   \n",
       "4   00005adc52a78e1630cfb50e4bd0f89e      67.150000             75.876154   \n",
       "5   000079a54709c3ce262a21327bebc5a7     251.216667            275.603389   \n",
       "6   0000824380faac87cc64aab251a65efd      36.200000             36.200000   \n",
       "7   0000839aee24e6f8d7856b7d5a89d665     113.683333            124.892909   \n",
       "8   000088d299916f2e44ef335f3752ae61      91.250000            103.286103   \n",
       "9   0000a3c4817080300cf3ef7d58faaad1      81.766667            105.369048   \n",
       "10  0000beca799d88e6810faf33df585dec      57.933333             61.766060   \n",
       "11  0000ce9bfa00bd78fc08e3747d1551d9      82.283333             89.695416   \n",
       "12  0000da1850824413bba84ada2269a89a      30.016667             30.016667   \n",
       "13  0000da877ccfd3045d8b7b524e5ea375     111.983333            129.300312   \n",
       "14  0000dbe2f9b7a09864e39eb56a415726     446.866667            499.270403   \n",
       "\n",
       "    inactive_time    target  \n",
       "0       22.101825  0.911212  \n",
       "1        0.000000  1.000000  \n",
       "2       93.049201  0.803695  \n",
       "3      387.263282  0.860005  \n",
       "4       42.173846  0.884995  \n",
       "5      268.563278  0.911515  \n",
       "6        0.000000  1.000000  \n",
       "7       52.873758  0.910246  \n",
       "8      186.080564  0.883468  \n",
       "9      351.497619  0.776003  \n",
       "10      30.933940  0.937948  \n",
       "11       7.821250  0.917364  \n",
       "12       0.000000  1.000000  \n",
       "13      14.649688  0.866072  \n",
       "14     148.612930  0.895039  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# When combine true is False, results match with Akshat \n",
    "target_df_new_decay_surv.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(186, 5)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_df_new_decay_surv[target_df_new_decay_surv['inactive_time'] < -1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# target = get_surv_prob(orders, save_file=False, use_cache=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# orders = merge_order_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# orders[orders.driver_id == '00032bc2d22da93ecfaa7b25a9a6f445'].sort_values('ride_start_timestamp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [df_final_old_decay, df_final_new_decay1, \n",
    "           df_final_new_decay10, df_final_new_decay100, df_final_new_decay_surv]\n",
    "target_df_list = [target_df_old_decay, target_df_new_decay1, \n",
    "                  target_df_new_decay10, target_df_new_decay100, target_df_new_decay_surv]\n",
    "name_list = ['no_decay', 'old_decay', 'new_decay1', 'new_decay10', 'new_decay100', 'survival']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [df_final_new_decay_surv]\n",
    "target_df_list = [target_df_new_decay_surv]\n",
    "name_list = ['survival']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no_decay 0.012954343981013843\n",
      "old_decay 0.013123681810831017\n",
      "new_decay1 0.017780472130803315\n",
      "new_decay10 0.021421235471872562\n",
      "new_decay100 0.0\n"
     ]
    }
   ],
   "source": [
    "for name, target in zip(name_list, target_df_list):\n",
    "    print(name, 100*target[target.target > 1].shape[0]/target.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " target_df_new_decay100[target_df_new_decay100.target > 1].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now to debug why greater than 0%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start = '2016-11-01'\n",
    "# end = '2016-11-30'\n",
    "# orders = merge_order_df(start=start, end=end, use_cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tmp = orders[['driver_id', 'ride_start_timestamp', 'ride_stop_timestamp', 'ride_duration']][orders.driver_id == '0001860739024029fa3da2cad0ed4de2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tmp['ride_duration'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tmp.sort_values('ride_start_timestamp').head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# actual_ride_duration = 53 + 32 + 31 + 58 + 48 + 27 + 22\n",
    "# actual_ride_duration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Printing results for 0001860739024029fa3da2cad0ed4de2\n",
    "\n",
    "# target_df = create_modified_active_time_through_decay2(orders, use_cache=False, save_file=False, mult_factor=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that ride duration is summed 3 times which have overlapping periods due to which it becomes greater than 1. Also as you can see above, the actual inactive time in row 33721 should be 28 but we make it 95 which cuts 60 minutes from the drives active time further increasing the error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# target_df[target_df.ride_duration > target_df.modified_active_time].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# val = ['0001860739024029fa3da2cad0ed4de2', '000211a97ba2b768ef941593bf1ae1a5', '0009cc512604d27cc3e819ab6b954fca', '3d5914545a97b002450e7769d7e51df3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# order_filter = orders[orders.driver_id.isin(val)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# order_filter.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now to see those who are active for more than 20 hours "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# target_df.sort_values('modified_active_time', ascending=False).head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Printing results for d18747ec3c039917014c1b397b528d97\n",
    "\n",
    "# target_df_new = create_modified_active_time_through_decay2(orders, use_cache=False, save_file=False, mult_factor=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "# orders[['ride_start_timestamp', 'ride_stop_timestamp', 'ride_duration']][orders.driver_id == 'd18747ec3c039917014c1b397b528d97'].sort_values('ride_start_timestamp')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In essence we are always saying a driver will be active between rides for 10 minutes at least "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modelling Code "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have till now coded the following models \n",
    "\n",
    "1. Linear Regression \n",
    "2. Linear Regression with L1 Regularization\n",
    "3. Linear Regression with L2 Regularization\n",
    "4. Random Forest Regressor "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we split the data in a ratio of 75:25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV, LassoCV, ElasticNet\n",
    "from sklearn.ensemble import RandomForestRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(df, target_df, name, seed=0):\n",
    "    # regression forest\n",
    "    print(f'building regression model for {name}')\n",
    "    X = df    \n",
    "    xtrain, xtest, ytrain, ytest = train_test_split(X, target_df['target'], random_state=seed)\n",
    "\n",
    "    sc = StandardScaler()\n",
    "\n",
    "    xtrain_sc = sc.fit_transform(xtrain)\n",
    "\n",
    "    rr = RandomForestRegressor(random_state=seed)\n",
    "    rr.fit(xtrain_sc, ytrain)\n",
    "\n",
    "    xtest_sc = sc.transform(xtest)\n",
    "    \n",
    "    print(rr.score(xtest_sc, ytest))\n",
    "    \n",
    "    return rr, rr.score(xtest_sc, ytest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final_new_decay_surv.shape, target_df_new_decay_surv.shape\n",
    "# df_final_no_decay.shape, target_df_no_decay.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [df_final_old_decay, df_final_new_decay1, \n",
    "           df_final_new_decay10, df_final_new_decay100, df_final_new_decay_surv]\n",
    "target_df_list = [target_df_old_decay, target_df_new_decay1, \n",
    "                  target_df_new_decay10, target_df_new_decay100, target_df_new_decay_surv]\n",
    "name_list = ['old_decay', 'new_decay1', 'new_decay10', 'new_decay100', 'survival']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_list = [new_df]\n",
    "# target_df_list = [new_target_df]\n",
    "# name_list = ['No_Pool']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building regression model for old_decay\n",
      "0.6711402299896342\n",
      "building regression model for new_decay1\n",
      "0.6615701030289285\n",
      "building regression model for new_decay10\n",
      "0.7426831032569411\n",
      "building regression model for new_decay100\n",
      "0.7932704036466552\n",
      "building regression model for survival\n",
      "0.5774645701649963\n"
     ]
    }
   ],
   "source": [
    "result = {}\n",
    "model = {}\n",
    "for df, target_df, name in zip(df_list, target_df_list, name_list):\n",
    "    model[name], result[name] = build_model(df, target_df, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_list = [df_final_new_decay_surv]\n",
    "target_df_list = [target_df_new_decay_surv]\n",
    "name_list = ['survival']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building regression model for survival\n",
      "0.5824010272026875\n"
     ]
    }
   ],
   "source": [
    "result = {}\n",
    "model = {}\n",
    "for df, target_df, name in zip(df_list, target_df_list, name_list):\n",
    "    model[name], result[name] = build_model(df, target_df, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>driver_id</th>\n",
       "      <th>ride_duration</th>\n",
       "      <th>survival_active_time</th>\n",
       "      <th>inactive_time</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0000131d486b69eb77ab6e9e7cca9f4c</td>\n",
       "      <td>75.583333</td>\n",
       "      <td>91.769072</td>\n",
       "      <td>13.280928</td>\n",
       "      <td>0.823625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>00002724a19c5f6a54ae8d60a378997e</td>\n",
       "      <td>34.800000</td>\n",
       "      <td>34.800000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>000042288ba964aa04d14869bd56efb6</td>\n",
       "      <td>73.083333</td>\n",
       "      <td>93.503874</td>\n",
       "      <td>90.479459</td>\n",
       "      <td>0.781608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>000049d47b79747ad9927ba1bf5f4126</td>\n",
       "      <td>188.766667</td>\n",
       "      <td>240.229715</td>\n",
       "      <td>374.803618</td>\n",
       "      <td>0.785776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>00005adc52a78e1630cfb50e4bd0f89e</td>\n",
       "      <td>67.150000</td>\n",
       "      <td>74.370283</td>\n",
       "      <td>43.679717</td>\n",
       "      <td>0.902914</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          driver_id  ride_duration  survival_active_time  \\\n",
       "0  0000131d486b69eb77ab6e9e7cca9f4c      75.583333             91.769072   \n",
       "1  00002724a19c5f6a54ae8d60a378997e      34.800000             34.800000   \n",
       "2  000042288ba964aa04d14869bd56efb6      73.083333             93.503874   \n",
       "3  000049d47b79747ad9927ba1bf5f4126     188.766667            240.229715   \n",
       "4  00005adc52a78e1630cfb50e4bd0f89e      67.150000             74.370283   \n",
       "\n",
       "   inactive_time    target  \n",
       "0      13.280928  0.823625  \n",
       "1       0.000000  1.000000  \n",
       "2      90.479459  0.781608  \n",
       "3     374.803618  0.785776  \n",
       "4      43.679717  0.902914  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_df_new_decay_surv.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_list = [df_final_new_decay100]\n",
    "# target_df_list = [target_df_new_decay100]\n",
    "# name_list = ['Decay100']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for df, target_df, name in zip(df_list, target_df_list, name_list):\n",
    "#     model[name], result[name] = build_model(df, target_df, name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list(new_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest = model['new_decay100']\n",
    "X = df_final_new_decay100\n",
    "importances = forest.feature_importances_\n",
    "std = np.std([tree.feature_importances_ for tree in forest.estimators_],\n",
    "             axis=0)\n",
    "indices = np.argsort(importances)[::-1]\n",
    "\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(X.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], importances[indices[f]]))\n",
    "\n",
    "# Plot the feature importances of the forest\n",
    "plt.figure()\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(X.shape[1]), importances[indices],\n",
    "       color=\"r\", yerr=std[indices], align=\"center\")\n",
    "plt.xticks(range(X.shape[1]), indices)\n",
    "plt.xlim([-1, X.shape[1]])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest = model['survival']\n",
    "X = df_final_new_decay_surv\n",
    "importances = forest.feature_importances_\n",
    "std = np.std([tree.feature_importances_ for tree in forest.estimators_],\n",
    "             axis=0)\n",
    "indices = np.argsort(importances)[::-1]\n",
    "\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(X.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], importances[indices[f]]))\n",
    "\n",
    "# Plot the feature importances of the forest\n",
    "plt.figure()\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(X.shape[1]), importances[indices],\n",
    "       color=\"r\", yerr=std[indices], align=\"center\")\n",
    "plt.xticks(range(X.shape[1]), indices)\n",
    "plt.xlim([-1, X.shape[1]])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# col = [x.shape[1] for x in df_list]\n",
    "# col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set(df_list[1].columns).difference(df_list[0].columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final_old_decay.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_final_new_decay10.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}