Category Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Production script for Classification\n",
    "\n",
    "This script highlights how the trained linear SVM model is to be loaded and run against the data of interest to classify text into their appropriate categories. The codes also identifies the location of event and the activity causing it where possible. The code assumes that the data is extracted in bulk in a CSV or Excel format with the column of interest labelled CONTENT. If the column is named otherwise, change the name accordingly throughout this script or in the data file (preferable). The processed data after classification will be re-saved in the original extension as the original file in the main directory as this script."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gensim\n",
    "from sklearn import utils\n",
    "import regex as re\n",
    "import string\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import SVC\n",
    "import dill as pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text preprocessing\n",
    "\n",
    "Below we define a function to convert text to lower-case and strip punctuation/symbols from words and so on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleanText(text):\n",
    "    text = re.sub(r'#', r'apartment unit ', text)\n",
    "    text = re.sub(r'['+string.punctuation+']', r\" \", text)\n",
    "    text = re.sub(r'[0-9]+', r' ', text)\n",
    "    text = re.sub('\\s+', \" \", text).strip()\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"\\bv deck\\b\", r\"voiddeck\", text)\n",
    "    text = re.sub(r\"\\bvd\\b\", r\"voiddeck\", text)\n",
    "    text = re.sub(r\"\\bv d\\b\", r\"voiddeck\", text)\n",
    "    text = re.sub(r\"\\bvdeck\\b\", r\"voiddeck\", text)\n",
    "    text = re.sub(r\"\\bvoid deck\\b\", r\"voiddeck\", text)\n",
    "    text = re.sub(r\"\\bfbck\\b\", r\"feedback\", text)\n",
    "    text = re.sub(r\"\\bfbk\\b\", r\"feedback\", text)\n",
    "    text = re.sub(r\"\\bf back\\b\", r\"feedback\", text)\n",
    "    text = re.sub(r\"\\bfback\\b\", r\"feedback\", text)\n",
    "    text = re.sub(r\"\\bfup\\b\", r\"feedback\", text)\n",
    "    text = re.sub(r\"\\bfb\\b\", r\"feedback\", text)\n",
    "    text = re.sub(r\"\\bres\\b\", r\"resident\", text)\n",
    "    text = re.sub(r\"\\bresd\\b\", r\"resident\", text)\n",
    "    text = re.sub(r\"\\bresid\\b\", r\"resident\", text)\n",
    "    text = re.sub(r\"\\breno\\b\", r\"renovation\", text)\n",
    "    text = re.sub(r\"\\bneighbour\\b\", r\"neighbor\", text)\n",
    "    text = re.sub(r\"\\bneighbours\\b\", r\"neighbor\", text)\n",
    "    text = re.sub(r\"\\bneighbors\\b\", r\"neighbor\", text)\n",
    "    text = re.sub(r\"\\breq\\b\", r\"request\", text)\n",
    "    text = re.sub(r\"\\brq\\b\", r\"request\", text)\n",
    "    text = re.sub(r\"\\binfd\\b\", r\"informed\", text)\n",
    "    text = re.sub(r\"\\badv\\b\", r\"advice\", text)\n",
    "    text = re.sub(r\"\\badvic\\b\", r\"advice\", text)\n",
    "    text = re.sub(r\"\\badvis\\b\", r\"advice\", text)\n",
    "    text = re.sub(r\"\\badvise\\b\", r\"advice\", text)\n",
    "    text = re.sub(r\"\\badviceeise\\b\", r\"advice\", text)\n",
    "    text = re.sub(r\"\\bppl\\b\", r\"people\", text)\n",
    "    text = re.sub(r\"\\bindians\\b\", r\"indian\", text)\n",
    "    text = re.sub(r\"\\bmalays\\b\", r\"malay\", text)\n",
    "    text = re.sub(r\"\\bment\\b\", r\"apartment\", text)\n",
    "    text = re.sub(r\"\\btcil\\b\", r\"towncouncil\", text)\n",
    "    text = re.sub(r\"\\btown council\\b\", r\"towncouncil\", text)\n",
    "    text = re.sub(r\"\\bcoffee shop\\b\", r\"coffeeshop\", text)\n",
    "    text = re.sub(r\"\\breno\\b\", r\"renovation\", text)\n",
    "    text = re.sub(r\"\\brenov\\b\", r\"renovation\", text)\n",
    "    text = re.sub(r\"\\bcpark\\b\", r\"carpark\", text)\n",
    "    text = re.sub(r\"\\bcar park\\b\", r\"carpark\", text)\n",
    "    text = re.sub(r\"\\bmscp\\b\", r\"multi-storey carpark\", text)\n",
    "    text = re.sub(r\"\\bmulti storey\\b\", r\"multi-storey\", text)\n",
    "    text = re.sub(r\"\\bmultistorey\\b\", r\"multi-storey\", text)\n",
    "    text = re.sub(r\"\\broad work\\b\", r\"roadworks\", text)\n",
    "    text = re.sub(r\"\\broad works\\b\", r\"roadworks\", text)\n",
    "    text = re.sub(r\"\\bfrm\\b\", r\"from\", text)\n",
    "    text = re.sub(r\"\\bgrp\\b\", r\"group\", text)\n",
    "    text = re.sub(r\"\\bhse\\b\", r\"house\", text)\n",
    "    text = re.sub(r\"\\bcan t\\b\", r\"cannot\", text)\n",
    "    text = re.sub(r\"\\bgrp\\b\", r\"group\", text)\n",
    "    text = re.sub(r\"\\bsaket bord\\b\", r\"skateboard\", text)\n",
    "    text = re.sub(r\"\\bw out\\b\", r\"without\", text)\n",
    "    text = re.sub(r\"\\bongg\\b\", r\"ongoing\", text)\n",
    "    text = re.sub(r\"\\bon gg\\b\", r\"ongoing\", text)\n",
    "    text = re.sub(r\"\\bfwake\\b\", r\"funeral wake\", text)\n",
    "    text = re.sub(r\"\\bcorr\\b\", r\"corridor\", text)\n",
    "    text = re.sub(r\"\\bcorri\\b\", r\"corridor\", text)\n",
    "    text = re.sub(r\"\\bcorridors\\b\", r\"corridor\", text)\n",
    "    text = re.sub(r\"\\bpg\\b\", r\"playground\", text)\n",
    "    text = re.sub(r\"\\bpgd\\b\", r\"playground\", text)\n",
    "    text = re.sub(r\"\\bpgrd\\b\", r\"playground\", text)\n",
    "    text = re.sub(r\"\\blway\\b\", r\"linkway\", text)\n",
    "    text = re.sub(r\"\\bl way\\b\", r\"linkway\", text)\n",
    "    text = re.sub(r\"\\bcontratcor\\b\", r\"linkway\", text)\n",
    "    text = re.sub(r\"\\bos\\b\", r\"outside\", text) \n",
    "    text = re.sub(r\"\\bo s\\b\", r\"outside\", text) \n",
    "    text = re.sub(r\"\\binfd\\b\", r\"informed\", text)\n",
    "    text = re.sub(r\"\\bfball\\b\", r\"football\", text)\n",
    "    text = re.sub(r\"\\bfoot ball\\b\", r\"football\", text)\n",
    "    text = re.sub(r\"\\brtop\\b\", r\"rooftop\", text)\n",
    "    text = re.sub(r\"\\br top\\b\", r\"rooftop\", text)\n",
    "    text = re.sub(r\"\\bramping\\b\", r\"revving\", text)\n",
    "    text = re.sub(r\"\\bramming\\b\", r\"revving\", text)\n",
    "    text = re.sub(r\"\\bcolln\\b\", r\"collection\", text)\n",
    "    text = re.sub(r\"\\bcontruction\\b\", r\"construction\", text)\n",
    "    text = re.sub(r\"\\bconctruction\\b\", r\"construction\", text)\n",
    "    text = re.sub(r\"\\bpurposed\\b\", r\"purpose\", text)\n",
    "    text = re.sub(r\"\\bmph\\b\", r\"multi-purpose hall\", text)\n",
    "    text = re.sub(r\"\\bmphall\\b\", r\"multi-purpose hall\", text)\n",
    "    text = re.sub(r\"\\bmulti purpose\\b\", r\"multi-purpose\", text)\n",
    "    text = re.sub(r\"\\bmultipurpose\\b\", r\"multi-purpose\", text)\n",
    "    text = re.sub(r\"\\bmulti purpos\\b\", r\"multi-purpose\", text)\n",
    "    text = re.sub(r\"\\bfurnitures\\b\", r\"furniture\", text)\n",
    "    text = re.sub(r\"\\bchairs\\b\", r\"chair\", text)\n",
    "    text = re.sub(r\"\\btables\\b\", r\"table\", text)\n",
    "    text = re.sub(r\"\\bcheers\\b\", r\"cheer\", text)\n",
    "    text = re.sub(r\"\\bsongs\\b\", r\"song\", text)\n",
    "    text = re.sub(r\"\\bsepak takraw\\b\", r\"sepaktakraw\", text)\n",
    "    text = re.sub(r\"\\bbadmointon\\b\", r\"badminton\", text)\n",
    "    text = re.sub(r\"\\bbadminton court\\b\", r\"badmintoncourt\", text)\n",
    "    text = re.sub(r\"\\bb minton\\b\", r\"badminton\", text)\n",
    "    text = re.sub(r\"\\bbasket ball\\b\", r\"basketball\", text)\n",
    "    text = re.sub(r\"\\bbasketball court\\b\", r\"basketballcourt\", text)\n",
    "    text = re.sub(r\"\\btakewondo\\b\", r\"taekwondo\", text)\n",
    "    text = re.sub(r\"\\btaichi\\b\", r\"tai chi\", text)\n",
    "    text = re.sub(r\"\\bgathing\\b\", r\"gathering\", text)\n",
    "    text = re.sub(r\"\\bsoccer court\\b\", r\"soccercourt\", text)\n",
    "    text = re.sub(r\"\\bfootball court\\b\", r\"footballcourt\", text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The function below tokenizes the text data in each row (document) into individual words. Sentences smaller than 3 words and words that correspond to stopwords are discarded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "stop_words = stopwords.words('english')\n",
    "\n",
    "def tokenize_text(text, stop_words):\n",
    "    tokens = []\n",
    "    for sent in nltk.sent_tokenize(text):\n",
    "        for word in nltk.word_tokenize(sent):\n",
    "            if len(word) < 3 or word in stop_words:\n",
    "                continue\n",
    "            tokens.append(word.lower())\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre-processing Text Data for classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>INFMT_TEL</th>\n",
       "      <th>CONTENT</th>\n",
       "      <th>INC_TYPE</th>\n",
       "      <th>REPORT_TIME</th>\n",
       "      <th>INC_LATITUDE</th>\n",
       "      <th>INC_LONGTITUDE</th>\n",
       "      <th>Postal Code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pOi2Y</td>\n",
       "      <td>A LOT OF DRUNK MAN MAKING NOISE. WANT TO SLEEP...</td>\n",
       "      <td>Noise Pollution</td>\n",
       "      <td>01/02/2018 00:01:53</td>\n",
       "      <td>1.445663</td>\n",
       "      <td>103.798129</td>\n",
       "      <td>730768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>qvMak</td>\n",
       "      <td>A GROUP OF TEENAGERS HAVE BEEN MAKING NOISE FO...</td>\n",
       "      <td>Noise Pollution</td>\n",
       "      <td>01/02/2018 00:03:03</td>\n",
       "      <td>1.366885</td>\n",
       "      <td>103.954505</td>\n",
       "      <td>510121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>+9sdG</td>\n",
       "      <td>A BUNCH OF DRUNK YOUNGSTERS IS MAKING SO MUCH ...</td>\n",
       "      <td>Noise Pollution</td>\n",
       "      <td>01/02/2018 00:03:04</td>\n",
       "      <td>1.436902</td>\n",
       "      <td>103.841032</td>\n",
       "      <td>761269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>plWnf</td>\n",
       "      <td>GROUP OF MOTORCYCLIST REVVING THEIR ENGINE.</td>\n",
       "      <td>Noise Pollution</td>\n",
       "      <td>01/02/2018 00:04:29</td>\n",
       "      <td>1.354098</td>\n",
       "      <td>103.958750</td>\n",
       "      <td>520382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5D1pB</td>\n",
       "      <td>SOMEONE PLAYING LOUD MUSIC OUTSIDE MY CONDO</td>\n",
       "      <td>Noise Pollution</td>\n",
       "      <td>01/02/2018 00:06:15</td>\n",
       "      <td>1.314288</td>\n",
       "      <td>103.890696</td>\n",
       "      <td>398026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>WuWwB</td>\n",
       "      <td>This KTV at Katong Plaza ground floor is very ...</td>\n",
       "      <td>Noise Pollution</td>\n",
       "      <td>01/02/2018 00:09:04</td>\n",
       "      <td>1.304163</td>\n",
       "      <td>103.903875</td>\n",
       "      <td>429979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>p5tM4</td>\n",
       "      <td>SOME IS PLAYING THE REMOTE CONTROL CAR AT THE ...</td>\n",
       "      <td>Noise Pollution</td>\n",
       "      <td>01/02/2018 00:09:31</td>\n",
       "      <td>1.387174</td>\n",
       "      <td>103.907775</td>\n",
       "      <td>540160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>oXXCA</td>\n",
       "      <td>Top floor neighbor making alot of noise, It se...</td>\n",
       "      <td>Noise Pollution</td>\n",
       "      <td>01/02/2018 00:12:34</td>\n",
       "      <td>1.323714</td>\n",
       "      <td>103.738649</td>\n",
       "      <td>600020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>+9sdG</td>\n",
       "      <td>OUR NEIGHBOR SON OPEN THE DOOR AND MAKING A LO...</td>\n",
       "      <td>Noise Pollution</td>\n",
       "      <td>01/02/2018 00:16:41</td>\n",
       "      <td>1.305550</td>\n",
       "      <td>103.863563</td>\n",
       "      <td>190010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>HzzhM</td>\n",
       "      <td>Group of people at the void deck very noisy, c...</td>\n",
       "      <td>Noise Pollution</td>\n",
       "      <td>01/02/2018 00:16:52</td>\n",
       "      <td>1.382495</td>\n",
       "      <td>103.764046</td>\n",
       "      <td>670543</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  INFMT_TEL                                            CONTENT  \\\n",
       "0     pOi2Y  A LOT OF DRUNK MAN MAKING NOISE. WANT TO SLEEP...   \n",
       "1     qvMak  A GROUP OF TEENAGERS HAVE BEEN MAKING NOISE FO...   \n",
       "2     +9sdG  A BUNCH OF DRUNK YOUNGSTERS IS MAKING SO MUCH ...   \n",
       "3     plWnf        GROUP OF MOTORCYCLIST REVVING THEIR ENGINE.   \n",
       "4     5D1pB        SOMEONE PLAYING LOUD MUSIC OUTSIDE MY CONDO   \n",
       "5     WuWwB  This KTV at Katong Plaza ground floor is very ...   \n",
       "6     p5tM4  SOME IS PLAYING THE REMOTE CONTROL CAR AT THE ...   \n",
       "7     oXXCA  Top floor neighbor making alot of noise, It se...   \n",
       "8     +9sdG  OUR NEIGHBOR SON OPEN THE DOOR AND MAKING A LO...   \n",
       "9     HzzhM  Group of people at the void deck very noisy, c...   \n",
       "\n",
       "          INC_TYPE          REPORT_TIME  INC_LATITUDE  INC_LONGTITUDE  \\\n",
       "0  Noise Pollution  01/02/2018 00:01:53      1.445663      103.798129   \n",
       "1  Noise Pollution  01/02/2018 00:03:03      1.366885      103.954505   \n",
       "2  Noise Pollution  01/02/2018 00:03:04      1.436902      103.841032   \n",
       "3  Noise Pollution  01/02/2018 00:04:29      1.354098      103.958750   \n",
       "4  Noise Pollution  01/02/2018 00:06:15      1.314288      103.890696   \n",
       "5  Noise Pollution  01/02/2018 00:09:04      1.304163      103.903875   \n",
       "6  Noise Pollution  01/02/2018 00:09:31      1.387174      103.907775   \n",
       "7  Noise Pollution  01/02/2018 00:12:34      1.323714      103.738649   \n",
       "8  Noise Pollution  01/02/2018 00:16:41      1.305550      103.863563   \n",
       "9  Noise Pollution  01/02/2018 00:16:52      1.382495      103.764046   \n",
       "\n",
       "   Postal Code  \n",
       "0       730768  \n",
       "1       510121  \n",
       "2       761269  \n",
       "3       520382  \n",
       "4       398026  \n",
       "5       429979  \n",
       "6       540160  \n",
       "7       600020  \n",
       "8       190010  \n",
       "9       670543  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# change below accordingly to where the file is located\n",
    "filename = './Raw Data/Noise_Data-20180201-20180430_with_postal_code.xlsx'\n",
    "\n",
    "if filename.lower().endswith('.csv'):\n",
    "    df = pd.read_csv(filename, encoding='latin1')\n",
    "elif filename.lower().endswith('.xlsx'):\n",
    "    df = pd.read_excel(filename)\n",
    "else:\n",
    "    raise ValueError('Wrong file format! Please use CSV or Excel.')\n",
    "\n",
    "# Display first 10 rows of the data\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Process the data in CONTENT and put the processed data in a new column\n",
    "df['CONTENT_clean'] = df['CONTENT'].apply(cleanText)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Group of people causing noise near the mookata shop. Please advise'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['CONTENT'].iloc[30] # view example of original data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'group of people causing noise near the mookata shop please advice'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['CONTENT_clean'].iloc[30] # view examples of the cleaned data (before removal of stopwords)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define locations of event and the activities causing it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tag locations and activity to complaints where possible\n",
    "location = [\"void deck\",\n",
    "            \"multi-purpose hall\",\n",
    "            \"multi-purpose court\",\n",
    "            \"multi-purpose plaza\",\n",
    "            \"corridor\",\n",
    "            \"basketballcourt\", \n",
    "            \"badmintoncourt\",\n",
    "            \"soccercourt\",\n",
    "            \"footballcourt\",\n",
    "            \"playground\",\n",
    "            \"carpark\",\n",
    "            \"multi-storey carpark\",\n",
    "            \"coffeeshop\"]\n",
    "\n",
    "activity = [\"mahjong\",\n",
    "            \"babies\", \n",
    "            \"crying\",\n",
    "            \"dismantling\",\n",
    "            \"dismantle\",\n",
    "            \"furniture\", \n",
    "            \"drag\",\n",
    "            \"hack\",\n",
    "            \"push\", \n",
    "            \"drag\",\n",
    "            \"kick\",            \n",
    "            \"bang\",\n",
    "            \"table\",\n",
    "            \"chairs\", \n",
    "            \"pull\",\n",
    "            \"shout\",\n",
    "            \"talk\",\n",
    "            \"cheer\",\n",
    "            \"play\",\n",
    "            \"dance\",\n",
    "            \"dancing\"\n",
    "            \"music\",\n",
    "            \"song\",\n",
    "            \"dance\",\n",
    "            \"sepaktakraw\",\n",
    "            \"skateboard\", \n",
    "            \"wedding\", \n",
    "            \"funeral\", \n",
    "            \"party\", \n",
    "            \"taekwondo\",\n",
    "            \"exercise\",\n",
    "            \"exercising\",\n",
    "            \"wushu\",\n",
    "            \"tai chi\"]\n",
    "\n",
    "activity2 = ['basketball',\n",
    "             'football',\n",
    "             'soccer',\n",
    "             'badminton']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/pandas/core/indexing.py:189: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0                                 \n",
       "1                                 \n",
       "2                                 \n",
       "3                                 \n",
       "4                                 \n",
       "5                                 \n",
       "6     carpark,multi-storey carpark\n",
       "7                                 \n",
       "8                                 \n",
       "9                                 \n",
       "10                                \n",
       "11                      coffeeshop\n",
       "12                      coffeeshop\n",
       "13                                \n",
       "14                                \n",
       "15                                \n",
       "16                                \n",
       "17                                \n",
       "18                                \n",
       "19                                \n",
       "20                                \n",
       "21                                \n",
       "22                                \n",
       "23                                \n",
       "24                                \n",
       "25                                \n",
       "26                                \n",
       "27                                \n",
       "28                                \n",
       "29                                \n",
       "Name: Location, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Location'] = \"\"\n",
    "\n",
    "location_lst = \"|\".join(location)\n",
    "\n",
    "ind = df.index[df['CONTENT_clean'].str.contains(location_lst)]\n",
    "\n",
    "for i in ind:\n",
    "    tmp = []\n",
    "    for word in location:\n",
    "        if word in df['CONTENT_clean'].loc[i]:\n",
    "            tmp.append(word)\n",
    "    if tmp:\n",
    "        df['Location'].loc[i] = ','.join(tmp)\n",
    "\n",
    "df['Location'].head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          \n",
       "1          \n",
       "2          \n",
       "3          \n",
       "4      play\n",
       "5          \n",
       "6      play\n",
       "7      bang\n",
       "8      talk\n",
       "9          \n",
       "10         \n",
       "11         \n",
       "12         \n",
       "13         \n",
       "14         \n",
       "15         \n",
       "16         \n",
       "17     talk\n",
       "18         \n",
       "19         \n",
       "20         \n",
       "21         \n",
       "22     bang\n",
       "23    party\n",
       "24         \n",
       "25         \n",
       "26         \n",
       "27         \n",
       "28         \n",
       "29         \n",
       "Name: Activity, dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['Activity'] = \"\"\n",
    "\n",
    "activity_lst = \"|\".join(activity)\n",
    "\n",
    "ind = df.index[df['CONTENT_clean'].str.contains(activity_lst)]\n",
    "\n",
    "for i in ind:\n",
    "    tmp = []\n",
    "    for word in activity:\n",
    "        if word in df['CONTENT_clean'].loc[i]:\n",
    "            tmp.append(word)\n",
    "    for word in activity2:\n",
    "        if re.search(r'\\b' + word + r'\\b', df['CONTENT_clean'].loc[i]):\n",
    "            tmp.append(word)\n",
    "    if tmp:\n",
    "        df['Activity'].loc[i] = ','.join(tmp)\n",
    "\n",
    "df['Activity'].head(30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenize the test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = []\n",
    "for i in range(df['CONTENT_clean'].shape[0]):\n",
    "    X_test.append(tokenize_text(df['CONTENT_clean'].iloc[i], stop_words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is what a data entry looks like after tokenization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['lot', 'drunk', 'man', 'making', 'noise', 'want', 'sleep', 'also']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Running classifer on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the model from disk\n",
    "svc_tfidf = pickle.load(open('noise_categories_model_svm_tfidf.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Category Prediction'] = svc_tfidf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.replace(np.nan, '', regex=True)\n",
    "\n",
    "#save data either to csv or excel depending on the original data file\n",
    "filename_s = os.path.splitext(os.path.basename(filename))[0] + ' Predicted Categories'\n",
    "if filename.lower().endswith('.csv'):\n",
    "    df.to_csv(filename_s + '.csv', index = False)\n",
    "else:\n",
    "    writer = pd.ExcelWriter(filename_s + '.xlsx')\n",
    "    df.to_excel(writer,'Sheet1',index=False)\n",
    "    writer.save()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}