NLP_Full.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy.lang.en import English"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reading In"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "major thank you to Mitch Murphy for his notebook detailing this all on kaggle - https://www.kaggle.com/mjmurphy28"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = Path(\"../Project_4/speeches_full\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_to_open = data_folder / \"Bush_1989.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('../Project_4/speeches_full/Bush_1989.txt')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_to_open"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(file_to_open)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = os.listdir('../Project_4/speeches_full/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(columns=['year', 'president', 'party', 'text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../Project_4/speeches_full/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "229"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'../Project_4/speeches_full/Reagan_1982.txt'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path + names[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(names)):\n",
    "    components = names[i].split('_')\n",
    "    name = components[0]\n",
    "    year = components[1].split('.')[0]\n",
    "    df.loc[i,'year'] = year\n",
    "    df.loc[i,'president'] = name \n",
    "    \n",
    "    file = data_folder / names[i]\n",
    "    text = open(file, 'rb')\n",
    "    \n",
    "    lines = str(text.read())\n",
    "    lines = lines.replace('\\\\n',' ')\n",
    "    df.loc[i, 'text'] = lines.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def year_convert(year):\n",
    "    return int(year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "invalid literal for int() with base 10: 'Store'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-456dcecd73f1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0myear\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0myear\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myear_convert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[1;32m   3846\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3847\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3848\u001b[0;31m                 \u001b[0mmapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3850\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/lib.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-c93390763fb8>\u001b[0m in \u001b[0;36myear_convert\u001b[0;34m(year)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0myear_convert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myear\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myear\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m: invalid literal for int() with base 10: 'Store'"
     ]
    }
   ],
   "source": [
    "df.year = df.year.apply(year_convert)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IF there is an issue with a certian year not working\n",
    "I ran into the issue of an entry being \"Store\" so I use this to hunt it down and remove it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[35]\n"
     ]
    }
   ],
   "source": [
    "print(df[df['year']=='Store'].index.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(df.index[35],inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Continue!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.year = df.year.apply(year_convert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>president</th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1982</td>\n",
       "      <td>Reagan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. speaker, mr. president, distinguished me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1902</td>\n",
       "      <td>Roosevelt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1914</td>\n",
       "      <td>Wilson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"gentlemen of the congress:  the session upon...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1911</td>\n",
       "      <td>Taft</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'part i  this message is the first of several...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1814</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1848</td>\n",
       "      <td>Polk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1836</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1969</td>\n",
       "      <td>Johnson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. speaker, mr. president, members of the c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1930</td>\n",
       "      <td>Hoover</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1854</td>\n",
       "      <td>Pierce</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1855</td>\n",
       "      <td>Pierce</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1931</td>\n",
       "      <td>Hoover</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1968</td>\n",
       "      <td>Johnson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. speaker, mr. president, members of the c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1815</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1910</td>\n",
       "      <td>Taft</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1837</td>\n",
       "      <td>Buren</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1915</td>\n",
       "      <td>Wilson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'gentlemen of the congress:  since i last had...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1903</td>\n",
       "      <td>Roosevelt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1983</td>\n",
       "      <td>Reagan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. speaker, mr. president, distinguished me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1901</td>\n",
       "      <td>Roosevelt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1912</td>\n",
       "      <td>Taft</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'part i  to the senate and house of represent...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1917</td>\n",
       "      <td>Wilson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'gentlemen of the congress:  eight months hav...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1835</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"fellow citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1856</td>\n",
       "      <td>Pierce</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1932</td>\n",
       "      <td>Hoover</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1834</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1816</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1916</td>\n",
       "      <td>Wilson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"gentlemen of the congress:  in fulfilling at...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1984</td>\n",
       "      <td>Reagan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. speaker, mr. president, distinguished me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1904</td>\n",
       "      <td>Roosevelt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1938</td>\n",
       "      <td>Roosevelt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. president, mr. speaker, members of the s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>1852</td>\n",
       "      <td>Fillmore</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>1812</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>1830</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1949</td>\n",
       "      <td>Truman</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. president, mr. speaker, members of the c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1853</td>\n",
       "      <td>Pierce</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1948</td>\n",
       "      <td>Truman</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"mr. president, mr. speaker, and members of t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>1831</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>1813</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1913</td>\n",
       "      <td>Wilson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'gentlemen of the congress:  in pursuance of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>1939</td>\n",
       "      <td>Roosevelt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. vice president, mr. speaker, members of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>1905</td>\n",
       "      <td>Roosevelt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1985</td>\n",
       "      <td>Reagan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. speaker, mr. president, distinguished me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>1889</td>\n",
       "      <td>Harrison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>1987</td>\n",
       "      <td>Reagan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. speaker, mr. president, distinguished me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1907</td>\n",
       "      <td>Roosevelt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1811</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>1851</td>\n",
       "      <td>Fillmore</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>1833</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow citizens of the senate and of the hou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>1963</td>\n",
       "      <td>Kennedy</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'mr. vice president, mr. speaker, members of ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    year  president party                                               text\n",
       "0   1982     Reagan   NaN  b'mr. speaker, mr. president, distinguished me...\n",
       "1   1902  Roosevelt   NaN  b'to the senate and house of representatives: ...\n",
       "2   1914     Wilson   NaN  b\"gentlemen of the congress:  the session upon...\n",
       "3   1911       Taft   NaN  b'part i  this message is the first of several...\n",
       "4   1814    Madison   NaN  b\"fellow-citizens of the senate and house of r...\n",
       "5   1848       Polk   NaN  b'fellow-citizens of the senate and of the hou...\n",
       "6   1836    Jackson   NaN  b'fellow citizens of the senate and of the hou...\n",
       "7   1969    Johnson   NaN  b'mr. speaker, mr. president, members of the c...\n",
       "8   1930     Hoover   NaN  b\"to the senate and house of representatives: ...\n",
       "9   1854     Pierce   NaN  b'fellow-citizens of the senate and of the hou...\n",
       "10  1855     Pierce   NaN  b'fellow-citizens of the senate and of the hou...\n",
       "11  1931     Hoover   NaN  b'to the senate and house of representatives: ...\n",
       "12  1968    Johnson   NaN  b'mr. speaker, mr. president, members of the c...\n",
       "13  1815    Madison   NaN  b'fellow-citizens of the senate and house of r...\n",
       "14  1910       Taft   NaN  b'to the senate and house of representatives: ...\n",
       "15  1837      Buren   NaN  b'fellow-citizens of the senate and house of r...\n",
       "16  1915     Wilson   NaN  b'gentlemen of the congress:  since i last had...\n",
       "17  1903  Roosevelt   NaN  b'to the senate and house of representatives: ...\n",
       "18  1983     Reagan   NaN  b'mr. speaker, mr. president, distinguished me...\n",
       "19  1901  Roosevelt   NaN  b'to the senate and house of representatives: ...\n",
       "20  1912       Taft   NaN  b'part i  to the senate and house of represent...\n",
       "21  1917     Wilson   NaN  b'gentlemen of the congress:  eight months hav...\n",
       "22  1835    Jackson   NaN  b\"fellow citizens of the senate and of the hou...\n",
       "23  1856     Pierce   NaN  b'fellow-citizens of the senate and of the hou...\n",
       "24  1932     Hoover   NaN  b'to the senate and house of representatives: ...\n",
       "25  1834    Jackson   NaN  b'fellow citizens of the senate and of the hou...\n",
       "26  1816    Madison   NaN  b'fellow-citizens of the senate and house of r...\n",
       "27  1916     Wilson   NaN  b\"gentlemen of the congress:  in fulfilling at...\n",
       "28  1984     Reagan   NaN  b'mr. speaker, mr. president, distinguished me...\n",
       "29  1904  Roosevelt   NaN  b'to the senate and house of representatives: ...\n",
       "30  1938  Roosevelt   NaN  b'mr. president, mr. speaker, members of the s...\n",
       "31  1852   Fillmore   NaN  b'fellow-citizens of the senate and of the hou...\n",
       "32  1812    Madison   NaN  b\"fellow-citizens of the senate and house of r...\n",
       "33  1830    Jackson   NaN  b'fellow citizens of the senate and of the hou...\n",
       "34  1949     Truman   NaN  b'mr. president, mr. speaker, members of the c...\n",
       "36  1853     Pierce   NaN  b'fellow-citizens of the senate and of the hou...\n",
       "37  1948     Truman   NaN  b\"mr. president, mr. speaker, and members of t...\n",
       "38  1831    Jackson   NaN  b'fellow citizens of the senate and of the hou...\n",
       "39  1813    Madison   NaN  b\"fellow-citizens of the senate and house of r...\n",
       "40  1913     Wilson   NaN  b'gentlemen of the congress:  in pursuance of ...\n",
       "41  1939  Roosevelt   NaN  b'mr. vice president, mr. speaker, members of ...\n",
       "42  1905  Roosevelt   NaN  b'to the senate and house of representatives: ...\n",
       "43  1985     Reagan   NaN  b'mr. speaker, mr. president, distinguished me...\n",
       "44  1889   Harrison   NaN  b'to the senate and house of representatives: ...\n",
       "45  1987     Reagan   NaN  b'mr. speaker, mr. president, distinguished me...\n",
       "46  1907  Roosevelt   NaN  b'to the senate and house of representatives: ...\n",
       "47  1811    Madison   NaN  b\"fellow-citizens of the senate and house of r...\n",
       "48  1851   Fillmore   NaN  b'fellow-citizens of the senate and of the hou...\n",
       "49  1833    Jackson   NaN  b'fellow citizens of the senate and of the hou...\n",
       "50  1963    Kennedy   NaN  b'mr. vice president, mr. speaker, members of ..."
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.sort_values('year')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df.reset_index(inplace=True)\n",
    "df = df.drop(['index'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>president</th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1790</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b''</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1791</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1792</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1793</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1794</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1795</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1796</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1797</td>\n",
       "      <td>Adams</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'gentlemen of the senate and gentlemen of the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1798</td>\n",
       "      <td>Adams</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'gentlemen of the senate and gentlemen of the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1799</td>\n",
       "      <td>Adams</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'gentlemen of the senate and gentlemen of the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1800</td>\n",
       "      <td>Adams</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'gentlemen of the senate and gentlemen of the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1801</td>\n",
       "      <td>Jefferson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1802</td>\n",
       "      <td>Jefferson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1803</td>\n",
       "      <td>Jefferson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'to the senate and house of representatives o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1804</td>\n",
       "      <td>Jefferson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'the senate and house of representatives of t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1805</td>\n",
       "      <td>Jefferson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'the senate and house of representatives of t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1806</td>\n",
       "      <td>Jefferson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'the senate and house of representatives of t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1807</td>\n",
       "      <td>Jefferson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"the senate and house of representatives of t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1808</td>\n",
       "      <td>Jefferson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"the senate and house of representatives of t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1809</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1810</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1811</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1812</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1813</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1814</td>\n",
       "      <td>Madison</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b\"fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    year   president party                                               text\n",
       "0   1790  Washington   NaN                                                b''\n",
       "1   1791  Washington   NaN  b'fellow-citizens of the senate and house of r...\n",
       "2   1792  Washington   NaN  b'fellow-citizens of the senate and house of r...\n",
       "3   1793  Washington   NaN  b'fellow-citizens of the senate and house of r...\n",
       "4   1794  Washington   NaN  b'fellow-citizens of the senate and house of r...\n",
       "5   1795  Washington   NaN  b'fellow-citizens of the senate and house of r...\n",
       "6   1796  Washington   NaN  b\"fellow-citizens of the senate and house of r...\n",
       "7   1797       Adams   NaN  b'gentlemen of the senate and gentlemen of the...\n",
       "8   1798       Adams   NaN  b'gentlemen of the senate and gentlemen of the...\n",
       "9   1799       Adams   NaN  b'gentlemen of the senate and gentlemen of the...\n",
       "10  1800       Adams   NaN  b'gentlemen of the senate and gentlemen of the...\n",
       "11  1801   Jefferson   NaN  b'fellow citizens of the senate and house of r...\n",
       "12  1802   Jefferson   NaN  b'to the senate and house of representatives: ...\n",
       "13  1803   Jefferson   NaN  b'to the senate and house of representatives o...\n",
       "14  1804   Jefferson   NaN  b'the senate and house of representatives of t...\n",
       "15  1805   Jefferson   NaN  b'the senate and house of representatives of t...\n",
       "16  1806   Jefferson   NaN  b'the senate and house of representatives of t...\n",
       "17  1807   Jefferson   NaN  b\"the senate and house of representatives of t...\n",
       "18  1808   Jefferson   NaN  b\"the senate and house of representatives of t...\n",
       "19  1809     Madison   NaN  b'fellow-citizens of the senate and house of r...\n",
       "20  1810     Madison   NaN  b'fellow-citizens of the senate and house of r...\n",
       "21  1811     Madison   NaN  b\"fellow-citizens of the senate and house of r...\n",
       "22  1812     Madison   NaN  b\"fellow-citizens of the senate and house of r...\n",
       "23  1813     Madison   NaN  b\"fellow-citizens of the senate and house of r...\n",
       "24  1814     Madison   NaN  b\"fellow-citizens of the senate and house of r..."
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Noting different names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>president</th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1790</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b''</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1791</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1792</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1793</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1794</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>b'fellow-citizens of the senate and house of r...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   year   president party                                               text\n",
       "0  1790  Washington   NaN                                                b''\n",
       "1  1791  Washington   NaN  b'fellow-citizens of the senate and house of r...\n",
       "2  1792  Washington   NaN  b'fellow-citizens of the senate and house of r...\n",
       "3  1793  Washington   NaN  b'fellow-citizens of the senate and house of r...\n",
       "4  1794  Washington   NaN  b'fellow-citizens of the senate and house of r..."
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# need to distinuish between Theodore Roosevelt and Franklin D. Roosevelt\n",
    "indices = df.query(\"president =='Roosevelt' & year <= 1909\").index\n",
    "df.loc[indices,'president'] = 'Theodore Roosevelt'\n",
    "\n",
    "indices = df.query(\"president == 'Roosevelt'\").index\n",
    "df.loc[indices,'president'] = 'Franklin D. Roosevelt'\n",
    "\n",
    "indices = df.query(\"president =='Bush' & year <= 1992\").index\n",
    "df.loc[indices,'president'] = 'George H. W. Bush'\n",
    "\n",
    "indices = df.query(\"president == 'Bush'\").index\n",
    "df.loc[indices,'president'] = 'George W. Bush'\n",
    "\n",
    "indices = df.query(\"president =='Johnson' & year <= 1869\").index\n",
    "df.loc[indices,'president'] = 'Andrew Johnson'\n",
    "\n",
    "indices = df.query(\"president == 'Johnson'\").index\n",
    "df.loc[indices,'president'] = 'Lyndon B. Johnson'\n",
    "\n",
    "indices = df.query(\"president =='Adams' & year <= 1801\").index\n",
    "df.loc[indices,'president'] = 'John Adams'\n",
    "\n",
    "indices = df.query(\"president == 'Adams'\").index\n",
    "df.loc[indices,'president'] = 'John Quincy Adams'\n",
    "\n",
    "\n",
    "indices = df.query(\"president =='Harrison' & year <= 1841\").index\n",
    "df.loc[indices,'president'] = 'William Henry Harrison'\n",
    "\n",
    "indices = df.query(\"president == 'Harrison'\").index\n",
    "df.loc[indices,'president'] = 'Benjamin Harrison'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fixing the text part"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def shaver(text):\n",
    "    return text[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.text = df.text.apply(shaver)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>president</th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1790</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>''</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1791</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1792</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1793</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1794</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   year   president party                                               text\n",
       "0  1790  Washington   NaN                                                 ''\n",
       "1  1791  Washington   NaN  'fellow-citizens of the senate and house of re...\n",
       "2  1792  Washington   NaN  'fellow-citizens of the senate and house of re...\n",
       "3  1793  Washington   NaN  'fellow-citizens of the senate and house of re...\n",
       "4  1794  Washington   NaN  'fellow-citizens of the senate and house of re..."
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>president</th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1791</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1792</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1793</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1794</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1795</td>\n",
       "      <td>Washington</td>\n",
       "      <td>NaN</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   year   president party                                               text\n",
       "1  1791  Washington   NaN  'fellow-citizens of the senate and house of re...\n",
       "2  1792  Washington   NaN  'fellow-citizens of the senate and house of re...\n",
       "3  1793  Washington   NaN  'fellow-citizens of the senate and house of re...\n",
       "4  1794  Washington   NaN  'fellow-citizens of the senate and house of re...\n",
       "5  1795  Washington   NaN  'fellow-citizens of the senate and house of re..."
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# need to drop George Washington's 1790 address as the file is empty\n",
    "df = df.iloc[1:,:]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding Parties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pres_to_party(name):\n",
    "    republican = ['Lincoln', 'Grant', 'Hayes', 'Garfield', 'Arthur', \n",
    "                  'Benjamin Harrison', 'McKinley', 'Theodore Roosevelt', \n",
    "                  'Taft', 'Harding', 'Coolidge', 'Hoover', 'Eisenhower', \n",
    "                  'Nixon', 'Ford', 'Reagan', 'George H. W. Bush', \n",
    "                  'George W. Bush', 'Trump']\n",
    "    if name in republican:\n",
    "        return 'Republican'\n",
    "    \n",
    "    democratic = ['Jackson', 'Buren', 'Polk', 'Pierce', \n",
    "                  'Buchanan', 'Cleveland', 'Wilson', 'Franklin D. Roosevelt', \n",
    "                  'Truman', 'Kennedy', 'Lyndon B. Johnson', 'Carter', 'Clinton', 'Obama']\n",
    "    if name in democratic:\n",
    "        return 'Democratic'\n",
    "    \n",
    "    whig = ['William Henry Harrison', 'Taylor', 'Fillmore']\n",
    "    if name in whig:\n",
    "        return 'Whig'\n",
    "    \n",
    "    national_union = ['Andrew Johnson']\n",
    "    if name in national_union:\n",
    "        return 'National Union'\n",
    "    \n",
    "    \n",
    "    unaffiliated = ['Washington', 'Tyler']\n",
    "    if name in unaffiliated:\n",
    "        return 'Unaffiliated'\n",
    "    \n",
    "    federalist = ['John Adams']\n",
    "    if name in federalist:\n",
    "        return 'Federalist'\n",
    "    \n",
    "    democratic_republican = ['Jefferson', 'Madison', 'Monroe', 'John Quincy Adams']\n",
    "    if name in democratic_republican:\n",
    "        return 'Democratic-Republican'\n",
    "    \n",
    "df.party = df.president.apply(pres_to_party)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Further Cleaning of Speeches "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['speech_clean'] = df['text']\n",
    "\n",
    "#Remove numbers\n",
    "alphanumeric = lambda x: re.sub('\\w*\\d\\w*', ' ', x)\n",
    "df['speech_clean'] = df['speech_clean'].apply(alphanumeric)\n",
    "\n",
    "#Remove punctuation\n",
    "punctuation = lambda x: re.sub('[%s]' % re.escape(string.punctuation), ' ', x)\n",
    "df['speech_clean'] = df['speech_clean'].apply(punctuation)\n",
    "\n",
    "#Remove non-breaking space\n",
    "xa0 = lambda x: re.sub('\\xa0', ' ', x)\n",
    "xad = lambda x: re.sub('\\xad', ' ', x)\n",
    "df['speech_clean'] = df['speech_clean'].apply(xa0).apply(xad)\n",
    "\n",
    "#Remove dash\n",
    "dash = lambda x: re.sub('–', ' ', x)\n",
    "df['speech_clean'] = df['speech_clean'].apply(dash)\n",
    "\n",
    "#Make lower case\n",
    "lower = lambda x: x.lower()\n",
    "df['speech_clean'] = df['speech_clean'].apply(lower)\n",
    "\n",
    "#Replace hyphens\n",
    "hyphen_1 = lambda x: re.sub('“', ' ', x)\n",
    "hyphen_2 = lambda x: re.sub('„', ' ', x)\n",
    "df['speech_clean'] = df['speech_clean'].apply(hyphen_1).apply(hyphen_2)\n",
    "\n",
    "#Remove doubles space\n",
    "spaces = lambda x: ' '.join(x.split())\n",
    "df['speech_clean'] = df['speech_clean'].apply(spaces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>president</th>\n",
       "      <th>party</th>\n",
       "      <th>text</th>\n",
       "      <th>speech_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1791</td>\n",
       "      <td>Washington</td>\n",
       "      <td>Unaffiliated</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "      <td>fellow citizens of the senate and house of rep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1792</td>\n",
       "      <td>Washington</td>\n",
       "      <td>Unaffiliated</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "      <td>fellow citizens of the senate and house of rep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1793</td>\n",
       "      <td>Washington</td>\n",
       "      <td>Unaffiliated</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "      <td>fellow citizens of the senate and house of rep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1794</td>\n",
       "      <td>Washington</td>\n",
       "      <td>Unaffiliated</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "      <td>fellow citizens of the senate and house of rep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1795</td>\n",
       "      <td>Washington</td>\n",
       "      <td>Unaffiliated</td>\n",
       "      <td>'fellow-citizens of the senate and house of re...</td>\n",
       "      <td>fellow citizens of the senate and house of rep...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   year   president         party  \\\n",
       "1  1791  Washington  Unaffiliated   \n",
       "2  1792  Washington  Unaffiliated   \n",
       "3  1793  Washington  Unaffiliated   \n",
       "4  1794  Washington  Unaffiliated   \n",
       "5  1795  Washington  Unaffiliated   \n",
       "\n",
       "                                                text  \\\n",
       "1  'fellow-citizens of the senate and house of re...   \n",
       "2  'fellow-citizens of the senate and house of re...   \n",
       "3  'fellow-citizens of the senate and house of re...   \n",
       "4  'fellow-citizens of the senate and house of re...   \n",
       "5  'fellow-citizens of the senate and house of re...   \n",
       "\n",
       "                                        speech_clean  \n",
       "1  fellow citizens of the senate and house of rep...  \n",
       "2  fellow citizens of the senate and house of rep...  \n",
       "3  fellow citizens of the senate and house of rep...  \n",
       "4  fellow citizens of the senate and house of rep...  \n",
       "5  fellow citizens of the senate and house of rep...  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_count(string):\n",
    "    '''counts number of words in speech'''\n",
    "    count = len(string.split())\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['word_count'] = df['speech_clean'].apply(word_count)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokens, Leminization vs Stem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import NMF\n",
    "from nltk.stem import WordNetLemmatizer \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stop Word Depot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop = stopwords.words('english')\n",
    "\n",
    "new_stop = ([\"'ll\", \"'re\", \"'ve\", 'could', 'doe', 'might', \n",
    "             'must', \"n't\", 'need', 'sha', 'would', 'abov', \n",
    "             'ask', 'drug', 'per', 'cent', 'my', 'member', \n",
    "             'ani', 'becaus', 'befor', 'dure', 'members',\n",
    "             'onc', 'onli', 'republican', 'thought', 'whole',\n",
    "             'ourselv', 'themselv', 'veri', 'whi', 'people', 'men',\n",
    "             'yourselv', 'tonight', 'let', 'program', 'purpose', \n",
    "             'shall', 'get', 'upon', 'like', 'ought', 'may', 'last', \n",
    "             'today', 'help', 'know', 'want', 'matter', 'report', 'president',\n",
    "            'present', 'say', 'purpos', 'purpo', 'year', 'america', 'new',\n",
    "             'american', 'federal', 'feder', 'speaker', 'vice', 'fellow',\n",
    "            'peopl', 'presid', 'great', 'gentleman', 'commissioner', 'object','subject',\n",
    "             'claim','mississippi','sum', 'person', 'commission', 'commiss',\n",
    "            'depart', 'offic'])\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "stop = stop + new_stop\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = WordNetLemmatizer() \n",
    "stemmer = SnowballStemmer('english')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'terrorism'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemmatizer.lemmatize('terrorism')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'terror'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemmatizer.lemmatize('terror')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'terror'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stemmer.stem('terror')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'terror'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stemmer.stem('terrorism')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Thank you to Metis alum Erik Hafner for this function\n",
    "https://github.com/ErikHafner "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def tokenize_and_lem(text):\n",
    "    '''applies tokenization and stemming'''\n",
    "    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]\n",
    "    filtered_tokens = []\n",
    "    for token in tokens:\n",
    "        token = lemmatizer.lemmatize(token)\n",
    "        if len(token) > 2:\n",
    "            filtered_tokens.append(token)\n",
    "    return filtered_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_topics(model, feature_names, no_top_words, topic_names=None):\n",
    "    '''shows topics identified in topic modeling'''\n",
    "    for ix, topic in enumerate(model.components_):\n",
    "        if not topic_names or not topic_names[ix]:\n",
    "            print(\"\\nTopic \", ix)\n",
    "        else:\n",
    "            print(\"\\nTopic: '\",topic_names[ix],\"'\")\n",
    "        print(\", \".join([feature_names[i]\n",
    "                        for i in topic.argsort()[:-no_top_words - 1:-1]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectorize me - lemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = TfidfVectorizer(stop_words=stop,tokenizer=tokenize_and_lem,min_df=.15,max_df=.9)\n",
    "sparse_tfidf = tfidf.fit_transform(df['speech_clean'])\n",
    "tfidf_feature_names = tfidf.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2604"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tfidf_feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nmf_10 = NMF(30,random_state=1)\n",
    "nmf_10_topic = nmf_10.fit_transform(sparse_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display_topics(nmf_10, tfidf_feature_names, 10, topic_names=None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectorize Me! - Stemmed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_and_stem(text):\n",
    "    '''applies tokenization and stemming'''\n",
    "    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]\n",
    "    filtered_tokens = []\n",
    "    for token in tokens:\n",
    "        token = stemmer.stem(token)\n",
    "        if len(token) > 2:\n",
    "            filtered_tokens.append(token)\n",
    "    return filtered_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = TfidfVectorizer(stop_words=stop,tokenizer=tokenize_and_stem,min_df=.20,max_df=.90)\n",
    "sparse_tfidf = tfidf.fit_transform(df['speech_clean'])\n",
    "tfidf_feature_names = tfidf.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1723"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tfidf_feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Topic  0\n",
      "indian, general, secretari, recommend, fiscal, pension, land, appropri, amount, june\n",
      "\n",
      "Topic  1\n",
      "children, budget, school, famili, challeng, parent, child, centuri, communiti, dream\n",
      "\n",
      "Topic  2\n",
      "agricultur, econom, industri, cooper, veteran, bank, problem, relief, construct, method\n",
      "\n",
      "Topic  3\n",
      "develop, econom, farm, world, recommend, billion, price, effort, achiev, million\n",
      "\n",
      "Topic  4\n",
      "fight, enemi, world, victori, product, air, alli, battl, german, freedom\n",
      "\n",
      "Topic  5\n",
      "corpor, man, forest, busi, court, condit, island, labor, navi, case\n",
      "\n",
      "Topic  6\n",
      "bank, duti, deposit, money, currenc, treasuri, general, institut, paper, speci\n",
      "\n",
      "Topic  7\n",
      "enemi, command, british, militia, lake, hostil, victori, general, success, britain\n",
      "\n",
      "Topic  8\n",
      "mexico, texa, mexican, territori, duti, treati, treasuri, california, minist, foreign\n",
      "\n",
      "Topic  9\n",
      "treati, relat, convent, intern, republ, consular, diplomat, foreign, arbitr, repres\n",
      "\n",
      "Topic  10\n",
      "job, busi, cut, colleg, worker, famili, economi, world, invest, back\n",
      "\n",
      "Topic  11\n",
      "coloni, duti, improv, amount, intercours, commerc, parti, consider, treasuri, vessel\n",
      "\n",
      "Topic  12\n",
      "tax, billion, spend, percent, budget, job, inflat, energi, cut, propos\n",
      "\n",
      "Topic  13\n",
      "soviet, world, free, defens, freedom, militari, econom, strength, threat, effort\n",
      "\n",
      "Topic  14\n",
      "spain, treati, articl, minist, majesti, territori, spanish, likewis, respect, appoint\n",
      "\n",
      "Topic  15\n",
      "constitut, territori, union, slave, general, republ, duti, thus, elect, question\n",
      "\n",
      "Topic  16\n",
      "minist, british, communic, council, instruct, french, commerc, franc, author, relat\n",
      "\n",
      "Topic  17\n",
      "necessari, railway, action, seem, duti, ship, develop, immedi, industri, counsel\n",
      "\n",
      "Topic  18\n",
      "vessel, million, sea, necessari, debt, within, whether, princip, harbor, call\n",
      "\n",
      "Topic  19\n",
      "terror, weapon, world, freedom, tax, children, health, women, fight, worker\n",
      "\n",
      "Topic  20\n",
      "indian, provis, general, consider, measur, execut, happi, attent, militia, tribe\n",
      "\n",
      "Topic  21\n",
      "democraci, problem, world, recoveri, faith, democrat, modern, unemploy, incom, form\n",
      "\n",
      "Topic  22\n",
      "tariff, manufactur, duti, price, articl, product, condit, bond, revenu, taxat\n",
      "\n",
      "Topic  23\n",
      "gold, silver, note, circul, currenc, treasuri, bond, amount, issu, valu\n",
      "\n",
      "Topic  24\n",
      "world, seek, goal, live, way, life, problem, come, better, group\n"
     ]
    }
   ],
   "source": [
    "nmf_10 = NMF(25,random_state=1)\n",
    "nmf_10_topic = nmf_10.fit_transform(sparse_tfidf)\n",
    "display_topics(nmf_10, tfidf_feature_names, 10, topic_names=None)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Topic Naming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "topic_names = (['Indian Issues',\n",
    "                'Education',\n",
    "                'Agriculture',\n",
    "                'Development',\n",
    "                'War',\n",
    "                'Corporations',\n",
    "                'Banking',\n",
    "                'Revolutionary War',\n",
    "                'Mexican War',\n",
    "                'Diplomacy',\n",
    "                'Employment',\n",
    "                'Colonies',\n",
    "                'Taxation',\n",
    "                'Communism',\n",
    "                'Spanish War',\n",
    "                'Civil War',\n",
    "                'British Conflict',\n",
    "                'Infrastructure',\n",
    "                'Naval Concerns',\n",
    "                'Terrorism',\n",
    "                'Indian Conflict',\n",
    "                'Democracy',\n",
    "                'Tariffs',\n",
    "                'Currency',\n",
    "                'Globalism'\n",
    "                ''\n",
    "                ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create dataframe with topic values\n",
    "H_35 = pd.DataFrame(nmf_10_topic.round(5), columns = topic_names)\n",
    "\n",
    "#merge dataframes\n",
    "df = pd.concat([df.reset_index(drop=True), H_35.reset_index(drop=True)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = df.drop(['text','speech_clean'] ,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lexical Richness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lex_rich(text):\n",
    "    return len(set(text)) / len(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['Lexical Richness'] = df['speech_clean']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1['Lexical Richness'] = df1['Lexical Richness'].apply(lex_rich)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save This One"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.to_csv('By_Year.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Speech Aggregation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get sum of topics per speech\n",
    "df['topic_sum']= 0\n",
    "for topic in topic_names:\n",
    "    df['topic_sum'] += df[topic]\n",
    "    \n",
    "#get percentage share of topic per speech\n",
    "aggregation_columns = topic_names + ['topic_sum']\n",
    "\n",
    "for topic in aggregation_columns:\n",
    "    df[topic] = df[topic]/df['topic_sum']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sum word count and topic columns for each president\n",
    "sum_columns = ['word_count'] + aggregation_columns\n",
    "\n",
    "df_speaker_parties = df.groupby(['president','party'])[sum_columns].apply(lambda x: x.sum()).reset_index()\n",
    "\n",
    "#get percentage share of topic per president\n",
    "for topic in topic_names:\n",
    "    df_speaker_parties[topic] = df_speaker_parties[topic]/df_speaker_parties['topic_sum']\n",
    "    \n",
    "df_speaker_parties.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_speaker_parties.head(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#run cluster analysis\n",
    "km_15 = KMeans(n_clusters=5,random_state=44)\n",
    "\n",
    "X_speaker = df_speaker_parties[topic_names].values\n",
    "km_15.fit(X_speaker)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_cluster(X,km=[],num_clusters=0):\n",
    "    color = 'brgcmyk'\n",
    "    alpha = 0.5\n",
    "    s = 20\n",
    "    if num_clusters == 0:\n",
    "        plt.scatter(X[:,0],X[:,1],c = color[0],alpha = alpha,s = s)\n",
    "    else:\n",
    "        for i in range(num_clusters):\n",
    "            plt.scatter(X[km.labels_==i,0],X[km.labels_==i,1],c = color[i],alpha = alpha,s=s)\n",
    "            plt.scatter(km.cluster_centers_[i][0],km.cluster_centers_[i][1],c = color[i], marker = 'x', s = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display_cluster(X_speaker,km_15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "km_15.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res=km_15.__dict__\n",
    "print(res['cluster_centers_'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_speaker_parties.to_csv('NLP_Full.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}