Modeling_Pass_Prasun.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-05T09:12:31.292544Z",
     "start_time": "2020-11-05T09:12:31.286663Z"
    }
   },
   "source": [
    "# Recommendation System - Model 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import packages and dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:45:23.311385Z",
     "start_time": "2020-12-17T01:45:21.699028Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random as rd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import OneHotEncoder as OHE\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "rd.seed(123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:46:09.451872Z",
     "start_time": "2020-12-17T01:45:23.313277Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/prastha/opt/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3147: DtypeWarning: Columns (4) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "# import all the files\n",
    "\n",
    "donationsDF = pd.read_csv('Donations.csv')\n",
    "donorsDF = pd.read_csv('Donors.csv')\n",
    "projectsDF = pd.read_csv('Projects.csv')\n",
    "schoolsDF = pd.read_csv('Schools.csv')\n",
    "resourcesDF = pd.read_csv('Resources.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Cleaning: Remove one-time donors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:46:24.950185Z",
     "start_time": "2020-12-17T01:46:09.596988Z"
    }
   },
   "outputs": [],
   "source": [
    "donorsDF_merged = donorsDF.merge(donationsDF, on = 'Donor ID', how = 'inner') # donation ID is the primary key here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:46:27.334738Z",
     "start_time": "2020-12-17T01:46:24.987267Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Donor ID</th>\n",
       "      <th>Num Donations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00000ce845c00cbf0686c992fc369df4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00002783bc5d108510f3f9666c8b1edd</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Donor ID  Num Donations\n",
       "0  00000ce845c00cbf0686c992fc369df4              1\n",
       "1  00002783bc5d108510f3f9666c8b1edd              1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get the count of donations a donor has made\n",
    "df_temp1 = donorsDF_merged.groupby(['Donor ID']).size().reset_index(name='Num Donations')\n",
    "df_temp2 = df_temp1[df_temp1['Num Donations'] > 1] # removing all the single-donors\n",
    "\n",
    "print(\"We got rid of %d single-donors.\" % (df_temp1.shape[0] - df_temp2.shape[0]))\n",
    "print(\"The shape of the remaining dataframe:\", df_temp2.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recurring donors for modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:46:30.804774Z",
     "start_time": "2020-12-17T01:46:27.442321Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3215610, 11)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniqDonorIDList = list(df_temp2['Donor ID']) # list of recurring donors\n",
    "df_final = donorsDF_merged[donorsDF_merged['Donor ID'].isin(uniqDonorIDList)] # recurring donoros\n",
    "\n",
    "del df_temp1, df_temp2, donorsDF_merged\n",
    "df_final.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:46:30.810650Z",
     "start_time": "2020-12-17T01:46:30.807281Z"
    }
   },
   "outputs": [],
   "source": [
    "def removeDuplicates(df, col):\n",
    "    df = df.drop_duplicates(subset = [col]) # removes duplicates from the column `col`\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:46:34.489068Z",
     "start_time": "2020-12-17T01:46:30.813130Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3215570, 11)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final = removeDuplicates(df_final, 'Donation ID') # removes duplicate Donation IDs\n",
    "\n",
    "# donation ID is the primary key here\n",
    "df_final = df_final.merge(projectsDF, on = \"Project ID\", how = \"inner\") # adding projects to the final dataframe\n",
    "\n",
    "df_final = df_final.merge(schoolsDF, on = \"School ID\", how = \"inner\") # adding schools to the final dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T02:32:26.380322Z",
     "start_time": "2020-12-17T02:32:15.502242Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3156862, 36)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Donor ID</th>\n",
       "      <th>Donor City</th>\n",
       "      <th>Donor State</th>\n",
       "      <th>Donor Is Teacher</th>\n",
       "      <th>Donor Zip</th>\n",
       "      <th>Project ID</th>\n",
       "      <th>Donation ID</th>\n",
       "      <th>Donation Included Optional Donation</th>\n",
       "      <th>Donation Amount</th>\n",
       "      <th>Donor Cart Sequence</th>\n",
       "      <th>...</th>\n",
       "      <th>Project Current Status</th>\n",
       "      <th>Project Fully Funded Date</th>\n",
       "      <th>School Name</th>\n",
       "      <th>School Metro Type</th>\n",
       "      <th>School Percentage Free Lunch</th>\n",
       "      <th>School State</th>\n",
       "      <th>School Zip</th>\n",
       "      <th>School City</th>\n",
       "      <th>School County</th>\n",
       "      <th>School District</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>666004</th>\n",
       "      <td>b2555e38cb649277fdbddc4c1acd290c</td>\n",
       "      <td>Boston</td>\n",
       "      <td>Massachusetts</td>\n",
       "      <td>Yes</td>\n",
       "      <td>021</td>\n",
       "      <td>21f9f1a452b90b08572c051ec4dcd29d</td>\n",
       "      <td>1147c2d6a73b69ac7ebc16dbf029b315</td>\n",
       "      <td>Yes</td>\n",
       "      <td>45.0</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2017-02-09</td>\n",
       "      <td>Curtis Guild Elementary School</td>\n",
       "      <td>urban</td>\n",
       "      <td>81.0</td>\n",
       "      <td>Massachusetts</td>\n",
       "      <td>2128</td>\n",
       "      <td>East Boston</td>\n",
       "      <td>Suffolk</td>\n",
       "      <td>Boston Public School District</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178482</th>\n",
       "      <td>770ab5fafe8342a82c8c9110692bc444</td>\n",
       "      <td>Charleston</td>\n",
       "      <td>South Carolina</td>\n",
       "      <td>Yes</td>\n",
       "      <td>294</td>\n",
       "      <td>bd5d1c0e5b3b58265726af22def13b81</td>\n",
       "      <td>570b4ed4da4d41ea1a8a0c08b2bb9f54</td>\n",
       "      <td>Yes</td>\n",
       "      <td>50.0</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2018-03-09</td>\n",
       "      <td>Burke High School</td>\n",
       "      <td>urban</td>\n",
       "      <td>85.0</td>\n",
       "      <td>South Carolina</td>\n",
       "      <td>29403</td>\n",
       "      <td>Charleston</td>\n",
       "      <td>Charleston</td>\n",
       "      <td>Charleston Co School District</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2073681</th>\n",
       "      <td>700ddab4673f35076f7e95631dccd55d</td>\n",
       "      <td>Grand Forks</td>\n",
       "      <td>North Dakota</td>\n",
       "      <td>No</td>\n",
       "      <td>582</td>\n",
       "      <td>1a67dfb38989e49ad744aedc856184cd</td>\n",
       "      <td>8bcd99179c69a2693d8a3f47707fd75c</td>\n",
       "      <td>Yes</td>\n",
       "      <td>100.0</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2016-08-26</td>\n",
       "      <td>Emerado Public School</td>\n",
       "      <td>rural</td>\n",
       "      <td>70.0</td>\n",
       "      <td>North Dakota</td>\n",
       "      <td>58228</td>\n",
       "      <td>Emerado</td>\n",
       "      <td>Grand Forks</td>\n",
       "      <td>Emerado School District 127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749250</th>\n",
       "      <td>de8b8ed34d1800ca93f7ef480857d872</td>\n",
       "      <td>Cleveland</td>\n",
       "      <td>Ohio</td>\n",
       "      <td>No</td>\n",
       "      <td>441</td>\n",
       "      <td>599253729f90d605c122b2d64863a90e</td>\n",
       "      <td>cc19bc57da9d55329707291f797bcba9</td>\n",
       "      <td>No</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1004</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2018-02-21</td>\n",
       "      <td>STEAM Academy</td>\n",
       "      <td>unknown</td>\n",
       "      <td>87.0</td>\n",
       "      <td>Missouri</td>\n",
       "      <td>63042</td>\n",
       "      <td>Hazelwood</td>\n",
       "      <td>St Louis</td>\n",
       "      <td>Ferguson-Florissant Sd R2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2152102</th>\n",
       "      <td>383df43a131fc39c8d8a59b2efcf1ebd</td>\n",
       "      <td>West Palm Beach</td>\n",
       "      <td>Florida</td>\n",
       "      <td>No</td>\n",
       "      <td>334</td>\n",
       "      <td>f25019f00401d05477abec0f427cae79</td>\n",
       "      <td>278d99217a3ffcc1ab5529110ebc6d99</td>\n",
       "      <td>Yes</td>\n",
       "      <td>50.0</td>\n",
       "      <td>44</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2018-03-27</td>\n",
       "      <td>Cholee Lake Elementary School</td>\n",
       "      <td>suburban</td>\n",
       "      <td>91.0</td>\n",
       "      <td>Florida</td>\n",
       "      <td>33413</td>\n",
       "      <td>Greenacres</td>\n",
       "      <td>Palm Beach</td>\n",
       "      <td>Palm Beach Co School District</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 Donor ID       Donor City     Donor State  \\\n",
       "666004   b2555e38cb649277fdbddc4c1acd290c           Boston   Massachusetts   \n",
       "178482   770ab5fafe8342a82c8c9110692bc444       Charleston  South Carolina   \n",
       "2073681  700ddab4673f35076f7e95631dccd55d      Grand Forks    North Dakota   \n",
       "749250   de8b8ed34d1800ca93f7ef480857d872        Cleveland            Ohio   \n",
       "2152102  383df43a131fc39c8d8a59b2efcf1ebd  West Palm Beach         Florida   \n",
       "\n",
       "        Donor Is Teacher Donor Zip                        Project ID  \\\n",
       "666004               Yes       021  21f9f1a452b90b08572c051ec4dcd29d   \n",
       "178482               Yes       294  bd5d1c0e5b3b58265726af22def13b81   \n",
       "2073681               No       582  1a67dfb38989e49ad744aedc856184cd   \n",
       "749250                No       441  599253729f90d605c122b2d64863a90e   \n",
       "2152102               No       334  f25019f00401d05477abec0f427cae79   \n",
       "\n",
       "                              Donation ID Donation Included Optional Donation  \\\n",
       "666004   1147c2d6a73b69ac7ebc16dbf029b315                                 Yes   \n",
       "178482   570b4ed4da4d41ea1a8a0c08b2bb9f54                                 Yes   \n",
       "2073681  8bcd99179c69a2693d8a3f47707fd75c                                 Yes   \n",
       "749250   cc19bc57da9d55329707291f797bcba9                                  No   \n",
       "2152102  278d99217a3ffcc1ab5529110ebc6d99                                 Yes   \n",
       "\n",
       "         Donation Amount  Donor Cart Sequence  ... Project Current Status  \\\n",
       "666004              45.0                    7  ...           Fully Funded   \n",
       "178482              50.0                    9  ...           Fully Funded   \n",
       "2073681            100.0                    4  ...           Fully Funded   \n",
       "749250               1.0                 1004  ...           Fully Funded   \n",
       "2152102             50.0                   44  ...           Fully Funded   \n",
       "\n",
       "        Project Fully Funded Date                     School Name  \\\n",
       "666004                 2017-02-09  Curtis Guild Elementary School   \n",
       "178482                 2018-03-09               Burke High School   \n",
       "2073681                2016-08-26           Emerado Public School   \n",
       "749250                 2018-02-21                   STEAM Academy   \n",
       "2152102                2018-03-27   Cholee Lake Elementary School   \n",
       "\n",
       "         School Metro Type School Percentage Free Lunch    School State  \\\n",
       "666004               urban                         81.0   Massachusetts   \n",
       "178482               urban                         85.0  South Carolina   \n",
       "2073681              rural                         70.0    North Dakota   \n",
       "749250             unknown                         87.0        Missouri   \n",
       "2152102           suburban                         91.0         Florida   \n",
       "\n",
       "        School Zip  School City School County                School District  \n",
       "666004        2128  East Boston       Suffolk  Boston Public School District  \n",
       "178482       29403   Charleston    Charleston  Charleston Co School District  \n",
       "2073681      58228      Emerado   Grand Forks    Emerado School District 127  \n",
       "749250       63042    Hazelwood      St Louis      Ferguson-Florissant Sd R2  \n",
       "2152102      33413   Greenacres    Palm Beach  Palm Beach Co School District  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# discretizing (binning) the project cost\n",
    "\n",
    "projCost = pd.DataFrame((df_final['Project Cost'] // 500 + 1) * 500).astype('int')\n",
    "\n",
    "projCost[projCost['Project Cost'] > 3000] = 3000\n",
    "\n",
    "projCost['Project Cost'] = projCost['Project Cost'].astype('str')\n",
    "projCost[projCost['Project Cost'] == '3000'] = \"3000+\"\n",
    "\n",
    "df_final['Project Cost'] = projCost\n",
    "df_final = df_final.rename(columns = {\"Project Cost\": \"Project Cost (Up to)\"})\n",
    "\n",
    "print(df_final.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T02:33:14.503802Z",
     "start_time": "2020-12-17T02:33:10.191289Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The original dataframe size:  (3156862, 36)\n",
      "The updated dataframe size including only California:  (479056, 36)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Donor ID</th>\n",
       "      <th>Donor City</th>\n",
       "      <th>Donor State</th>\n",
       "      <th>Donor Is Teacher</th>\n",
       "      <th>Donor Zip</th>\n",
       "      <th>Project ID</th>\n",
       "      <th>Donation ID</th>\n",
       "      <th>Donation Included Optional Donation</th>\n",
       "      <th>Donation Amount</th>\n",
       "      <th>Donor Cart Sequence</th>\n",
       "      <th>...</th>\n",
       "      <th>Project Current Status</th>\n",
       "      <th>Project Fully Funded Date</th>\n",
       "      <th>School Name</th>\n",
       "      <th>School Metro Type</th>\n",
       "      <th>School Percentage Free Lunch</th>\n",
       "      <th>School State</th>\n",
       "      <th>School Zip</th>\n",
       "      <th>School City</th>\n",
       "      <th>School County</th>\n",
       "      <th>School District</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6964</th>\n",
       "      <td>8f4befc408d5fb279be9e32611e0e0b7</td>\n",
       "      <td>Palo Alto</td>\n",
       "      <td>California</td>\n",
       "      <td>No</td>\n",
       "      <td>943</td>\n",
       "      <td>233a24939a563317cf242eb271f0d157</td>\n",
       "      <td>7b8c28289fd207c74928b72d1ee03979</td>\n",
       "      <td>No</td>\n",
       "      <td>99.0</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2015-08-23</td>\n",
       "      <td>Costano School-49Ers Academy</td>\n",
       "      <td>suburban</td>\n",
       "      <td>92.0</td>\n",
       "      <td>California</td>\n",
       "      <td>94303</td>\n",
       "      <td>E Palo Alto</td>\n",
       "      <td>San Mateo</td>\n",
       "      <td>Ravenswood City Elem Sch Dist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2394914</th>\n",
       "      <td>8e802e1abaa9d6f0cb58c26bc16c2a54</td>\n",
       "      <td>Mission Viejo</td>\n",
       "      <td>California</td>\n",
       "      <td>No</td>\n",
       "      <td>926</td>\n",
       "      <td>c993ac9f7fd87570f3da7a695a47e4b7</td>\n",
       "      <td>abf819a6b7010d55ffcf5f37795decd4</td>\n",
       "      <td>Yes</td>\n",
       "      <td>25.0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2016-09-29</td>\n",
       "      <td>La Paz Intermediate School</td>\n",
       "      <td>suburban</td>\n",
       "      <td>21.0</td>\n",
       "      <td>California</td>\n",
       "      <td>92691</td>\n",
       "      <td>Mission Viejo</td>\n",
       "      <td>Orange</td>\n",
       "      <td>Saddleback Valley Unified Sd</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 Donor ID     Donor City Donor State  \\\n",
       "6964     8f4befc408d5fb279be9e32611e0e0b7      Palo Alto  California   \n",
       "2394914  8e802e1abaa9d6f0cb58c26bc16c2a54  Mission Viejo  California   \n",
       "\n",
       "        Donor Is Teacher Donor Zip                        Project ID  \\\n",
       "6964                  No       943  233a24939a563317cf242eb271f0d157   \n",
       "2394914               No       926  c993ac9f7fd87570f3da7a695a47e4b7   \n",
       "\n",
       "                              Donation ID Donation Included Optional Donation  \\\n",
       "6964     7b8c28289fd207c74928b72d1ee03979                                  No   \n",
       "2394914  abf819a6b7010d55ffcf5f37795decd4                                 Yes   \n",
       "\n",
       "         Donation Amount  Donor Cart Sequence  ... Project Current Status  \\\n",
       "6964                99.0                    9  ...           Fully Funded   \n",
       "2394914             25.0                    2  ...           Fully Funded   \n",
       "\n",
       "        Project Fully Funded Date                   School Name  \\\n",
       "6964                   2015-08-23  Costano School-49Ers Academy   \n",
       "2394914                2016-09-29    La Paz Intermediate School   \n",
       "\n",
       "         School Metro Type School Percentage Free Lunch School State  \\\n",
       "6964              suburban                         92.0   California   \n",
       "2394914           suburban                         21.0   California   \n",
       "\n",
       "        School Zip    School City School County                School District  \n",
       "6964         94303    E Palo Alto     San Mateo  Ravenswood City Elem Sch Dist  \n",
       "2394914      92691  Mission Viejo        Orange   Saddleback Valley Unified Sd  \n",
       "\n",
       "[2 rows x 36 columns]"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#we are only considering all the donations that came to projects in California\n",
    "\n",
    "df_final = removeDuplicates(df_final, 'Donation ID') # removes duplicate Donation IDs\n",
    "print(\"The original dataframe size: \", df_final.shape)\n",
    "\n",
    "df_final_cal = df_final[df_final['School State'] == 'California']\n",
    "print(\"The updated dataframe size including only California: \", df_final_cal.shape)\n",
    "\n",
    "df_final_cal.sample(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training and Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T02:33:33.958322Z",
     "start_time": "2020-12-17T02:33:31.257796Z"
    }
   },
   "outputs": [],
   "source": [
    "columnsList = ['Donor ID', 'Donation Received Date', 'Donor City', 'Donor State',\n",
    "       'Donor Is Teacher', 'Donor Zip', 'Project ID', 'Donation ID',\n",
    "       'Donation Included Optional Donation', 'Donation Amount',\n",
    "       'Donor Cart Sequence', 'School ID', 'Teacher ID',\n",
    "       'Teacher Project Posted Sequence', 'Project Type', 'Project Title',\n",
    "       'Project Essay', 'Project Short Description', 'Project Need Statement',\n",
    "       'Project Subject Category Tree', 'Project Subject Subcategory Tree',\n",
    "       'Project Grade Level Category', 'Project Resource Category',\n",
    "       'Project Cost (Up to)', 'Project Posted Date', 'Project Expiration Date',\n",
    "       'Project Current Status', 'Project Fully Funded Date', 'School Name',\n",
    "       'School Metro Type', 'School Percentage Free Lunch', 'School State',\n",
    "       'School Zip', 'School City', 'School County', 'School District']\n",
    "\n",
    "df_final_cal = df_final_cal[columnsList]\n",
    "\n",
    "df_final_cal = df_final_cal.sort_values(by = ['Donor ID', 'Donation Received Date']) # sorts dataframe by donors and their\n",
    "                                                                            # first donation date\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T02:35:07.199494Z",
     "start_time": "2020-12-17T02:35:06.697899Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Donor ID</th>\n",
       "      <th>Donation Received Date</th>\n",
       "      <th>Donor City</th>\n",
       "      <th>Donor State</th>\n",
       "      <th>Donor Is Teacher</th>\n",
       "      <th>Donor Zip</th>\n",
       "      <th>Project ID</th>\n",
       "      <th>Donation ID</th>\n",
       "      <th>Donation Included Optional Donation</th>\n",
       "      <th>Donation Amount</th>\n",
       "      <th>...</th>\n",
       "      <th>Project Current Status</th>\n",
       "      <th>Project Fully Funded Date</th>\n",
       "      <th>School Name</th>\n",
       "      <th>School Metro Type</th>\n",
       "      <th>School Percentage Free Lunch</th>\n",
       "      <th>School State</th>\n",
       "      <th>School Zip</th>\n",
       "      <th>School City</th>\n",
       "      <th>School County</th>\n",
       "      <th>School District</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1267688</th>\n",
       "      <td>cca23670ed0d03bddfcd9c16fa9dacb7</td>\n",
       "      <td>2012-12-17 11:22:56</td>\n",
       "      <td>Las Vegas</td>\n",
       "      <td>Nevada</td>\n",
       "      <td>No</td>\n",
       "      <td>891</td>\n",
       "      <td>05d7c3d0703441e6b16ba5dfdd878c47</td>\n",
       "      <td>e353b95ffda5f4650224b2f638721c26</td>\n",
       "      <td>Yes</td>\n",
       "      <td>100.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2013-02-28</td>\n",
       "      <td>Oliveira Elementary School</td>\n",
       "      <td>suburban</td>\n",
       "      <td>20.0</td>\n",
       "      <td>California</td>\n",
       "      <td>94536</td>\n",
       "      <td>Fremont</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Fremont Unif School District</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2758782</th>\n",
       "      <td>8d345d4ded01a6c7f3b79e9686dfe741</td>\n",
       "      <td>2012-12-17 19:56:44</td>\n",
       "      <td>Oakland</td>\n",
       "      <td>California</td>\n",
       "      <td>Yes</td>\n",
       "      <td>946</td>\n",
       "      <td>4f921539c0a89df04b8a946fc31fb9b2</td>\n",
       "      <td>d2acc567eb8e09ccf87f85e8505d149f</td>\n",
       "      <td>Yes</td>\n",
       "      <td>20.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2013-08-29</td>\n",
       "      <td>Oakland Charter High School</td>\n",
       "      <td>urban</td>\n",
       "      <td>79.0</td>\n",
       "      <td>California</td>\n",
       "      <td>94601</td>\n",
       "      <td>Oakland</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Oakland Unified School Dist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2758777</th>\n",
       "      <td>45c45882b12febaf50ea7d25c464c33c</td>\n",
       "      <td>2012-12-19 18:23:19</td>\n",
       "      <td>San Bruno</td>\n",
       "      <td>California</td>\n",
       "      <td>No</td>\n",
       "      <td>940</td>\n",
       "      <td>4f921539c0a89df04b8a946fc31fb9b2</td>\n",
       "      <td>9cc5978ca69e102cea481a5db08a7f87</td>\n",
       "      <td>Yes</td>\n",
       "      <td>25.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2013-08-29</td>\n",
       "      <td>Oakland Charter High School</td>\n",
       "      <td>urban</td>\n",
       "      <td>79.0</td>\n",
       "      <td>California</td>\n",
       "      <td>94601</td>\n",
       "      <td>Oakland</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Oakland Unified School Dist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2758776</th>\n",
       "      <td>2ec1193b14119b158c1d3bfd67b2940b</td>\n",
       "      <td>2012-12-22 02:03:39</td>\n",
       "      <td>Oakland</td>\n",
       "      <td>California</td>\n",
       "      <td>No</td>\n",
       "      <td>946</td>\n",
       "      <td>4f921539c0a89df04b8a946fc31fb9b2</td>\n",
       "      <td>9d84a921ab8616dfad88ef15206bb2de</td>\n",
       "      <td>Yes</td>\n",
       "      <td>25.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2013-08-29</td>\n",
       "      <td>Oakland Charter High School</td>\n",
       "      <td>urban</td>\n",
       "      <td>79.0</td>\n",
       "      <td>California</td>\n",
       "      <td>94601</td>\n",
       "      <td>Oakland</td>\n",
       "      <td>Alameda</td>\n",
       "      <td>Oakland Unified School Dist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1704614</th>\n",
       "      <td>cc9cf018f0b68657c0ab7891df16c0fe</td>\n",
       "      <td>2013-01-01 14:35:18</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>California</td>\n",
       "      <td>Yes</td>\n",
       "      <td>941</td>\n",
       "      <td>7045b057d157ea53de41f5207ded655d</td>\n",
       "      <td>2e4ad8076f94da8b945e44c8fa8b9bb8</td>\n",
       "      <td>No</td>\n",
       "      <td>25.0</td>\n",
       "      <td>...</td>\n",
       "      <td>Fully Funded</td>\n",
       "      <td>2013-02-18</td>\n",
       "      <td>Lafayette Elementary School</td>\n",
       "      <td>urban</td>\n",
       "      <td>33.0</td>\n",
       "      <td>California</td>\n",
       "      <td>94121</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>San Francisco</td>\n",
       "      <td>San Francisco Unified Sch Dist</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 Donor ID Donation Received Date  \\\n",
       "1267688  cca23670ed0d03bddfcd9c16fa9dacb7    2012-12-17 11:22:56   \n",
       "2758782  8d345d4ded01a6c7f3b79e9686dfe741    2012-12-17 19:56:44   \n",
       "2758777  45c45882b12febaf50ea7d25c464c33c    2012-12-19 18:23:19   \n",
       "2758776  2ec1193b14119b158c1d3bfd67b2940b    2012-12-22 02:03:39   \n",
       "1704614  cc9cf018f0b68657c0ab7891df16c0fe    2013-01-01 14:35:18   \n",
       "\n",
       "            Donor City Donor State Donor Is Teacher Donor Zip  \\\n",
       "1267688      Las Vegas      Nevada               No       891   \n",
       "2758782        Oakland  California              Yes       946   \n",
       "2758777      San Bruno  California               No       940   \n",
       "2758776        Oakland  California               No       946   \n",
       "1704614  San Francisco  California              Yes       941   \n",
       "\n",
       "                               Project ID                       Donation ID  \\\n",
       "1267688  05d7c3d0703441e6b16ba5dfdd878c47  e353b95ffda5f4650224b2f638721c26   \n",
       "2758782  4f921539c0a89df04b8a946fc31fb9b2  d2acc567eb8e09ccf87f85e8505d149f   \n",
       "2758777  4f921539c0a89df04b8a946fc31fb9b2  9cc5978ca69e102cea481a5db08a7f87   \n",
       "2758776  4f921539c0a89df04b8a946fc31fb9b2  9d84a921ab8616dfad88ef15206bb2de   \n",
       "1704614  7045b057d157ea53de41f5207ded655d  2e4ad8076f94da8b945e44c8fa8b9bb8   \n",
       "\n",
       "        Donation Included Optional Donation  Donation Amount  ...  \\\n",
       "1267688                                 Yes            100.0  ...   \n",
       "2758782                                 Yes             20.0  ...   \n",
       "2758777                                 Yes             25.0  ...   \n",
       "2758776                                 Yes             25.0  ...   \n",
       "1704614                                  No             25.0  ...   \n",
       "\n",
       "         Project Current Status Project Fully Funded Date  \\\n",
       "1267688            Fully Funded                2013-02-28   \n",
       "2758782            Fully Funded                2013-08-29   \n",
       "2758777            Fully Funded                2013-08-29   \n",
       "2758776            Fully Funded                2013-08-29   \n",
       "1704614            Fully Funded                2013-02-18   \n",
       "\n",
       "                         School Name  School Metro Type  \\\n",
       "1267688   Oliveira Elementary School           suburban   \n",
       "2758782  Oakland Charter High School              urban   \n",
       "2758777  Oakland Charter High School              urban   \n",
       "2758776  Oakland Charter High School              urban   \n",
       "1704614  Lafayette Elementary School              urban   \n",
       "\n",
       "        School Percentage Free Lunch School State School Zip    School City  \\\n",
       "1267688                         20.0   California      94536        Fremont   \n",
       "2758782                         79.0   California      94601        Oakland   \n",
       "2758777                         79.0   California      94601        Oakland   \n",
       "2758776                         79.0   California      94601        Oakland   \n",
       "1704614                         33.0   California      94121  San Francisco   \n",
       "\n",
       "         School County                 School District  \n",
       "1267688        Alameda    Fremont Unif School District  \n",
       "2758782        Alameda     Oakland Unified School Dist  \n",
       "2758777        Alameda     Oakland Unified School Dist  \n",
       "2758776        Alameda     Oakland Unified School Dist  \n",
       "1704614  San Francisco  San Francisco Unified Sch Dist  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "donorsKeep = df_final_cal.drop_duplicates(subset=['Donor ID'], keep='first') # keep just the first donation for each donor\n",
    "donorsKeep = donorsKeep.sort_values(['Donation Received Date']) # sort the donation by donation date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T02:33:42.724592Z",
     "start_time": "2020-12-17T02:33:42.720834Z"
    }
   },
   "outputs": [],
   "source": [
    "donationsTrainNum = int(round(0.7 * len(donorsKeep), 0)) # the top ~70% is the training data\n",
    "\n",
    "donationsTrain = donorsKeep.iloc[:donationsTrainNum,:] # training data\n",
    "donationsValid = donorsKeep.iloc[donationsTrainNum:,] # the bottom ~30% is the validation data\n",
    "\n",
    "del donorsKeep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T02:33:47.643360Z",
     "start_time": "2020-12-17T02:33:47.616311Z"
    }
   },
   "outputs": [],
   "source": [
    "donorsTrainList = list(donationsTrain['Donor ID'])\n",
    "donorsValidList = list(donationsValid['Donor ID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T02:33:54.036066Z",
     "start_time": "2020-12-17T02:33:51.883257Z"
    }
   },
   "outputs": [],
   "source": [
    "donationsTrainFinal = df_final_cal[df_final_cal['Donor ID'].isin(donorsTrainList)] # expanding the training data\n",
    "donationsValidFinal = df_final_cal[df_final_cal['Donor ID'].isin(donorsValidList)] # expanding the validation data\n",
    "\n",
    "donationsTrainFinal = donationsTrainFinal.sort_values(by = ['Donor ID', 'Donation Received Date']) \n",
    "donationsValidFinal = donationsValidFinal.sort_values(by = ['Donor ID', 'Donation Received Date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T02:33:55.434400Z",
     "start_time": "2020-12-17T02:33:54.806937Z"
    }
   },
   "outputs": [],
   "source": [
    "# remove duplicates\n",
    "donationsTrainFinal = removeDuplicates(donationsTrainFinal, 'Donation ID') # donation ID is the primary key here\n",
    "donationsValidFinal = removeDuplicates(donationsValidFinal, 'Donation ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T02:33:56.724417Z",
     "start_time": "2020-12-17T02:33:56.720256Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data shape: \n",
      " (379181, 36)\n",
      "Validation data shape: \n",
      " (99875, 36)\n",
      "Percentage of training data:  0.7915170668982332\n"
     ]
    }
   ],
   "source": [
    "print(\"Training data shape: \\n\", donationsTrainFinal.shape)\n",
    "print(\"Validation data shape: \\n\", donationsValidFinal.shape)\n",
    "\n",
    "print(\"Percentage of training data: \", donationsTrainFinal.shape[0] / len(df_final_cal))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:48:15.049050Z",
     "start_time": "2020-12-17T01:48:15.037685Z"
    }
   },
   "outputs": [],
   "source": [
    "del donationsDF, donorsDF, projectsDF, schoolsDF, resourcesDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the recommendation system\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T02:41:57.458282Z",
     "start_time": "2020-12-17T02:41:57.451215Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# because we are doing content-based filtering, here are the features of the projects we are interested in:\n",
    "\n",
    "ohFeatures = ['Project ID', 'Project Expiration Date', 'Project Cost (Up to)', 'Project Type','Project Subject Category Tree',\n",
    "       'Project Grade Level Category', 'Project Resource Category',\n",
    "       'Project Current Status',\n",
    "       'School Metro Type', 'School State']\n",
    "\n",
    "# features we will do one hot encoding on\n",
    "dropCol = ['Project Cost (Up to)', 'Project Type', 'Project Subject Category Tree', 'Project Grade Level Category',\n",
    "           'Project Resource Category', 'Project Current Status',\n",
    "           'School Metro Type', 'School State']\n",
    "\n",
    "def oheDataFrame(dataframe): # the dataframe could either be training or validation set\n",
    "    projFeatures = dataframe[ohFeatures] # only considering the selected features (ohFeatures)\n",
    "    projFeatures = removeDuplicates(projFeatures, 'Project ID')\n",
    "    projFeatures = projFeatures.dropna() # drop the null values as well\n",
    "    projFeatures = projFeatures.sort_values(by = ['Project ID'])\n",
    "\n",
    "    featuresList = list(projFeatures.columns)\n",
    "    \n",
    "    enc = OHE(handle_unknown = 'ignore')\n",
    "    enc.fit(projFeatures.iloc[:,2:]) # fit the one hot vector on our dataframe\n",
    "    \n",
    "    # convert the one-hot matrix into a dataframe\n",
    "    oneHotMatrix = pd.DataFrame(enc.transform(projFeatures.iloc[:,2:]).toarray())\n",
    "    projOHM = pd.concat([projFeatures.reset_index(drop = True), oneHotMatrix.reset_index(drop = True)], axis = 1)\n",
    "    projOHM = projOHM.drop(dropCol, axis = 1) # dropping the categorical columns because they have already been vectorized\n",
    "\n",
    "    # data dictionary to track the one hot matrix\n",
    "\n",
    "    dataDict = list(enc.get_feature_names(dropCol)) # please note that any new input should strictly should the indexes\n",
    "                                                    # mentioned in data dictionary\n",
    "\n",
    "    \n",
    "    # scale and normalize the dataset\n",
    "\n",
    "    ohmDF = pd.DataFrame(StandardScaler().fit_transform(projOHM.iloc[:,2:]))\n",
    "    ohmDFwProj = pd.concat([projOHM[['Project ID', 'Project Expiration Date']], ohmDF], axis = 1)\n",
    "    ohmDFwProj['Project Expiration Date'] =  pd.to_datetime(ohmDFwProj['Project Expiration Date'])\n",
    "    \n",
    "    return ohmDFwProj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:47:11.256364Z",
     "start_time": "2020-12-17T01:47:11.252299Z"
    }
   },
   "outputs": [],
   "source": [
    "# this functions returns all the project that the donor has donated to\n",
    "# input is a donor ID (a string)\n",
    "# output is an array of all the donated projects\n",
    "\n",
    "def donatedProjects(main_dataframe, donorID):\n",
    "    df = main_dataframe[main_dataframe['Donor ID'] == donorID]\n",
    "    df = df.sort_values(['Donation Received Date']) # sort the donated projects by date (in ascending order)\n",
    "    df = df.drop_duplicates(subset = ['Project ID']) # remove repetitive donations on the same project\n",
    "    donatedProjsList = np.array(df['Project ID']) # we only need the donated projects\n",
    "    \n",
    "    return donatedProjsList\n",
    "\n",
    "# test the function here\n",
    "# donatedProjects(donationsTrainFinal, 'd0dc7dcaae4b97cd465ec7b53c86cd6a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:47:11.269586Z",
     "start_time": "2020-12-17T01:47:11.266162Z"
    }
   },
   "outputs": [],
   "source": [
    "# this function returns the one-hot encoded project features for a given project ID\n",
    "# input is a project ID\n",
    "# output is an array of the project features\n",
    "\n",
    "def projectFeatures(projDF, projectID):\n",
    "    feat = np.array(projDF[projDF['Project ID'] == projectID]) #projDF is the database of the project featuers\n",
    "    \n",
    "    return feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:47:11.277678Z",
     "start_time": "2020-12-17T01:47:11.271887Z"
    }
   },
   "outputs": [],
   "source": [
    "def cosineSimilarity(projDF, obv, numRec): #only works for one observation (project) at a time\n",
    "\n",
    "    project = obv[0][2:] # project features\n",
    "    project = project.reshape(1,len(project)) # convert 1D to 2D array\n",
    "    \n",
    "    projectID = obv[0][0]\n",
    "    projectDate = obv[0][1]\n",
    "    \n",
    "    # getting the cosine similarity between our feature matrix and test sample\n",
    "    cosSim = cosine_similarity(projDF.iloc[:,2:], project) # one-hot encoded dataframe is the dataframe here\n",
    "    recProjects = pd.concat([projDF.iloc[:,:2], pd.DataFrame(cosSim)], axis = 1)\n",
    "\n",
    "    recProjects = recProjects[recProjects['Project Expiration Date'] > projectDate] # only recommend projects that expire after the donation date\n",
    "    \n",
    "    recProjects = recProjects.sort_values(by = 0, ascending = False) # sorting the similarity in descending order\n",
    "    top10Rec = list(recProjects.head(numRec + 1).iloc[1:,0]) # the first column is the project ID\n",
    "    \n",
    "    return top10Rec\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:47:11.284179Z",
     "start_time": "2020-12-17T01:47:11.279993Z"
    }
   },
   "outputs": [],
   "source": [
    "def randomRecommendation(dataframe, first_proj, numRec):\n",
    "    feature = 'Project ID'\n",
    "    projects = removeDuplicates(dataframe, feature)\n",
    "    projects = list(projects['Project ID'])\n",
    "    \n",
    "    random = True\n",
    "    while random:\n",
    "        recProjs = rd.sample(projects, numRec)\n",
    "        if first_proj not in recProjs:\n",
    "            random = False\n",
    "    \n",
    "    return recProjs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:47:11.290382Z",
     "start_time": "2020-12-17T01:47:11.286458Z"
    }
   },
   "outputs": [],
   "source": [
    "def precisionRandomRec(main_dataframe, donorID, numRec):\n",
    "    donated_proj = donatedProjects(main_dataframe, donorID) # all the projects the donor has donated to\n",
    "                                                            # main dataframe here is either the final training or validation dataframe \n",
    "    \n",
    "    first_donation = donated_proj[0] # first donation of the donor\n",
    "    recProjs = randomRecommendation(main_dataframe, first_donation, numRec)\n",
    "    hits = 0\n",
    "\n",
    "    for i in donated_proj[1:]:\n",
    "        if i in recProjs:\n",
    "            hits += 1\n",
    "\n",
    "    return hits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T01:47:11.296914Z",
     "start_time": "2020-12-17T01:47:11.292695Z"
    }
   },
   "outputs": [],
   "source": [
    "def getTopPrecision(main_dataframe, projectsDF, donorID, numRec):\n",
    "    donated_proj = donatedProjects(main_dataframe, donorID) # all the projects the donor has donated to\n",
    "                                                            # main dataframe here is either the final training or validation dataframe \n",
    "    \n",
    "    first_donation = donated_proj[0] # first donation of the donor\n",
    "    \n",
    "    \n",
    "    proj_features = projectFeatures(projectsDF, first_donation)\n",
    "    hits = 0\n",
    "    rec_projs = cosineSimilarity(projectsDF, proj_features, numRec) # recommendations based on the first donation\n",
    "\n",
    "    for i in donated_proj[1:]:\n",
    "        if i in rec_projs:\n",
    "            hits += 1\n",
    "\n",
    "    return hits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T03:52:49.648571Z",
     "start_time": "2020-12-17T03:52:49.642854Z"
    }
   },
   "outputs": [],
   "source": [
    "# this function retrieves the precision of training and validation data as well as random recommendation system\n",
    "\n",
    "def getPrecision(donor_list, precision_type, main_dataframe, ohe_dataframe):\n",
    "    # donor_list is the list of donors in either training or validation dataset\n",
    "    # precision_type is the type for which the recommendation system is called\n",
    "    # main_dataframe is the original training or validation df\n",
    "    # ohe_dataframe is the OHE main_dataframe\n",
    "    \n",
    "    rd.seed(123)\n",
    "    \n",
    "    num_recs = [5, 10, 25, 50, 100] # number of recommendations\n",
    "    training_datapoints = 1000\n",
    "    hits = 0\n",
    "    precision_results = {}\n",
    "    \n",
    "    for i in num_recs:\n",
    "        for j in rd.sample(donor_list, training_datapoints):\n",
    "            if precision_type == 'random':\n",
    "                hits += precisionRandomRec(main_dataframe, j, i)\n",
    "            elif precision_type == 'training' or precision_type == 'validation':\n",
    "                hits += getTopPrecision(main_dataframe, ohe_dataframe, j, i)\n",
    "    \n",
    "        precision = hits / (i * training_datapoints)\n",
    "        precision_results[i] = precision\n",
    "    \n",
    "    return precision_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recommendation System Execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T03:57:55.322822Z",
     "start_time": "2020-12-17T03:57:52.868868Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{5: 0.0}"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calling all the functions above\n",
    "\n",
    "ohe_training_df = oheDataFrame(donationsTrainFinal) # projects in the training dataset\n",
    "ohe_validation_df = oheDataFrame(donationsValidFinal) # projects in the validation dataset\n",
    "\n",
    "# random recommendation system for the validation dataset\n",
    "randomrec_precision_results = getPrecision(donorsValidList, 'random', donationsValidFinal, ohe_validation_df)\n",
    "\n",
    "# precision results for training dataset\n",
    "train_rec_precision_results = getPrecision(donorsTrainList, 'training', donationsTrainFinal, ohe_training_df)\n",
    "valid_rec_precision_results = getPrecision(donorsValidList, 'validation', donationsValidFinal, ohe_validation_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T04:04:58.074846Z",
     "start_time": "2020-12-17T04:04:58.071392Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{5: 0, 10: 0, 25: 0, 50: 0, 100: 0}\n",
      "{5: 0.0008, 10: 0.0011, 25: 0.00116, 50: 0.00092, 100: 0.00068}\n",
      "{5: 0.002, 10: 0.0014, 25: 0.00088, 50: 0.00082, 100: 0.00061}\n"
     ]
    }
   ],
   "source": [
    "print(randomrec_precision_results)\n",
    "print(train_rec_precision_results)\n",
    "print(valid_rec_precision_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T04:28:42.783031Z",
     "start_time": "2020-12-17T04:28:42.778532Z"
    }
   },
   "outputs": [],
   "source": [
    "randomrec_results = pd.DataFrame.from_dict(randomrec_precision_results, orient = 'index').reset_index()\n",
    "randomrec_results.columns = ['# of Recommendations', 'Random Recommendation Precision']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T04:29:51.534154Z",
     "start_time": "2020-12-17T04:29:51.517310Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set Results\n",
      "--------------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th># of Recommendations</th>\n",
       "      <th>Training Precision</th>\n",
       "      <th>Training Lower Bound</th>\n",
       "      <th>Training Upper Bound</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>0.080</td>\n",
       "      <td>0.072480</td>\n",
       "      <td>0.087520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>0.110</td>\n",
       "      <td>0.103867</td>\n",
       "      <td>0.116133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>25</td>\n",
       "      <td>0.116</td>\n",
       "      <td>0.112030</td>\n",
       "      <td>0.119970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50</td>\n",
       "      <td>0.092</td>\n",
       "      <td>0.089467</td>\n",
       "      <td>0.094533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100</td>\n",
       "      <td>0.068</td>\n",
       "      <td>0.066440</td>\n",
       "      <td>0.069560</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   # of Recommendations  Training Precision  Training Lower Bound  \\\n",
       "0                     5               0.080              0.072480   \n",
       "1                    10               0.110              0.103867   \n",
       "2                    25               0.116              0.112030   \n",
       "3                    50               0.092              0.089467   \n",
       "4                   100               0.068              0.066440   \n",
       "\n",
       "   Training Upper Bound  \n",
       "0              0.087520  \n",
       "1              0.116133  \n",
       "2              0.119970  \n",
       "3              0.094533  \n",
       "4              0.069560  "
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Training Set Results\")\n",
    "print(\"----\" * 5)\n",
    "\n",
    "rec_train_results = pd.DataFrame.from_dict(train_rec_precision_results, orient = 'index').reset_index()\n",
    "rec_train_results.columns = ['# of Recommendations', 'Training Precision']\n",
    "rec_train_results['Training Precision'] = rec_train_results['Training Precision'] * 100\n",
    "\n",
    "rec_train_results['Training Lower Bound'] = rec_train_results['Training Precision'] - 1.96 * np.sqrt(rec_train_results['Training Precision'] * (1 - rec_train_results['Training Precision']) / (rec_train_results['# of Recommendations'] * 1000))\n",
    "rec_train_results['Training Upper Bound'] = rec_train_results['Training Precision'] + 1.96 * np.sqrt(rec_train_results['Training Precision'] * (1 - rec_train_results['Training Precision']) / (rec_train_results['# of Recommendations'] * 1000))\n",
    "\n",
    "rec_train_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T04:29:52.255015Z",
     "start_time": "2020-12-17T04:29:52.238545Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set Results\n",
      "--------------------\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th># of Recommendations</th>\n",
       "      <th>Validation Precision</th>\n",
       "      <th>Validation Lower Bound</th>\n",
       "      <th>Validation Upper Bound</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.188913</td>\n",
       "      <td>0.211087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>0.140</td>\n",
       "      <td>0.133199</td>\n",
       "      <td>0.146801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>25</td>\n",
       "      <td>0.088</td>\n",
       "      <td>0.084488</td>\n",
       "      <td>0.091512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50</td>\n",
       "      <td>0.082</td>\n",
       "      <td>0.079595</td>\n",
       "      <td>0.084405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100</td>\n",
       "      <td>0.061</td>\n",
       "      <td>0.059517</td>\n",
       "      <td>0.062483</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   # of Recommendations  Validation Precision  Validation Lower Bound  \\\n",
       "0                     5                 0.200                0.188913   \n",
       "1                    10                 0.140                0.133199   \n",
       "2                    25                 0.088                0.084488   \n",
       "3                    50                 0.082                0.079595   \n",
       "4                   100                 0.061                0.059517   \n",
       "\n",
       "   Validation Upper Bound  \n",
       "0                0.211087  \n",
       "1                0.146801  \n",
       "2                0.091512  \n",
       "3                0.084405  \n",
       "4                0.062483  "
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Validation Set Results\")\n",
    "print(\"----\" * 5)\n",
    "\n",
    "rec_val_results = pd.DataFrame.from_dict(valid_rec_precision_results, orient = 'index').reset_index()\n",
    "rec_val_results.columns = ['# of Recommendations', 'Validation Precision']\n",
    "rec_val_results['Validation Precision'] = rec_val_results['Validation Precision'] * 100\n",
    "\n",
    "rec_val_results['Validation Lower Bound'] = rec_val_results['Validation Precision'] - 1.96 * np.sqrt(rec_val_results['Validation Precision'] * (1 - rec_val_results['Validation Precision']) / (rec_val_results['# of Recommendations'] * 1000))\n",
    "rec_val_results['Validation Upper Bound'] = rec_val_results['Validation Precision'] + 1.96 * np.sqrt(rec_val_results['Validation Precision'] * (1 - rec_val_results['Validation Precision']) / (rec_val_results['# of Recommendations'] * 1000))\n",
    "\n",
    "rec_val_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-12-17T04:30:56.271375Z",
     "start_time": "2020-12-17T04:30:56.253523Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th># of Recommendations</th>\n",
       "      <th>Random Recommendation Precision</th>\n",
       "      <th>Training Precision</th>\n",
       "      <th>Training Lower Bound</th>\n",
       "      <th>Training Upper Bound</th>\n",
       "      <th>Validation Precision</th>\n",
       "      <th>Validation Lower Bound</th>\n",
       "      <th>Validation Upper Bound</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0.080</td>\n",
       "      <td>0.072480</td>\n",
       "      <td>0.087520</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.188913</td>\n",
       "      <td>0.211087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0.110</td>\n",
       "      <td>0.103867</td>\n",
       "      <td>0.116133</td>\n",
       "      <td>0.140</td>\n",
       "      <td>0.133199</td>\n",
       "      <td>0.146801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>0.116</td>\n",
       "      <td>0.112030</td>\n",
       "      <td>0.119970</td>\n",
       "      <td>0.088</td>\n",
       "      <td>0.084488</td>\n",
       "      <td>0.091512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>0.092</td>\n",
       "      <td>0.089467</td>\n",
       "      <td>0.094533</td>\n",
       "      <td>0.082</td>\n",
       "      <td>0.079595</td>\n",
       "      <td>0.084405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>0.068</td>\n",
       "      <td>0.066440</td>\n",
       "      <td>0.069560</td>\n",
       "      <td>0.061</td>\n",
       "      <td>0.059517</td>\n",
       "      <td>0.062483</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   # of Recommendations  Random Recommendation Precision  Training Precision  \\\n",
       "0                     5                                0               0.080   \n",
       "1                    10                                0               0.110   \n",
       "2                    25                                0               0.116   \n",
       "3                    50                                0               0.092   \n",
       "4                   100                                0               0.068   \n",
       "\n",
       "   Training Lower Bound  Training Upper Bound  Validation Precision  \\\n",
       "0              0.072480              0.087520                 0.200   \n",
       "1              0.103867              0.116133                 0.140   \n",
       "2              0.112030              0.119970                 0.088   \n",
       "3              0.089467              0.094533                 0.082   \n",
       "4              0.066440              0.069560                 0.061   \n",
       "\n",
       "   Validation Lower Bound  Validation Upper Bound  \n",
       "0                0.188913                0.211087  \n",
       "1                0.133199                0.146801  \n",
       "2                0.084488                0.091512  \n",
       "3                0.079595                0.084405  \n",
       "4                0.059517                0.062483  "
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_final = randomrec_results.merge(rec_train_results, on = '# of Recommendations').merge(rec_val_results, on = '# of Recommendations')\n",
    "results_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "228.821px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}