log_regr_classifier_davidson_et_al.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ei4AcgfpV1kd"
   },
   "source": [
    "# Replication for results in Davidson et al. 2017. \"Automated Hate Speech Detection and the Problem of Offensive Language\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sNw6YZi6qK3g"
   },
   "source": [
    "## Importing Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 158
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 19031,
     "status": "ok",
     "timestamp": 1588672457781,
     "user": {
      "displayName": "Samridha Man Shrestha",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjWM1giZm-4hM3_aud8FxlOUc8VWuoYWktFbs7VDA=s64",
      "userId": "14722659845832153120"
     },
     "user_tz": -240
    },
    "id": "4QTEqXzVV1ke",
    "outputId": "a152e09a-9912-4db8-d7cf-b7e40d4b4d02"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /Users/sam/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /Users/sam/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import sys\n",
    "import nltk\n",
    "import string\n",
    "import seaborn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Note tqdm import try\n",
    "try:\n",
    "    from tqdm.notebook import tqdm\n",
    "except:\n",
    "    import tqdm\n",
    "\n",
    "import textstat\n",
    "from nltk.stem.porter import *\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer as VS\n",
    "\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "nltk.download('stopwords')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /Users/sam/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /Users/sam/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "from util.preprocess_log_regr_and_svm import preprocess\n",
    "from util.preprocess_log_regr_and_svm import tokenize\n",
    "from util.preprocess_log_regr_and_svm import basic_tokenize\n",
    "from util.preprocess_log_regr_and_svm import count_twitter_objs\n",
    "from util.preprocess_log_regr_and_svm import other_features\n",
    "from util.preprocess_log_regr_and_svm import get_feature_array\n",
    "from util.preprocess_log_regr_and_svm import generate_features_train_data\n",
    "from util.preprocess_log_regr_and_svm import generate_features_test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NeUyr7qaV1ki"
   },
   "source": [
    "## Loading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 198
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4000,
     "status": "ok",
     "timestamp": 1588673159937,
     "user": {
      "displayName": "Samridha Man Shrestha",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjWM1giZm-4hM3_aud8FxlOUc8VWuoYWktFbs7VDA=s64",
      "userId": "14722659845832153120"
     },
     "user_tz": -240
    },
    "id": "lhk18szsV1ki",
    "outputId": "819df64c-cec4-4657-fc97-ed5d9f656335"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>count</th>\n",
       "      <th>hate_speech</th>\n",
       "      <th>offensive_language</th>\n",
       "      <th>neither</th>\n",
       "      <th>class</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>!!! RT @mayasolovely: As a woman you shouldn't...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!! RT @mleew17: boy dats cold...tyga dwn ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!!!! RT @C_G_Anderson: @viva_based she lo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  count  hate_speech  offensive_language  neither  class  \\\n",
       "0           0      3            0                   0        3      2   \n",
       "1           1      3            0                   3        0      1   \n",
       "2           2      3            0                   3        0      1   \n",
       "3           3      3            0                   2        1      1   \n",
       "4           4      6            0                   6        0      1   \n",
       "\n",
       "                                               tweet  \n",
       "0  !!! RT @mayasolovely: As a woman you shouldn't...  \n",
       "1  !!!!! RT @mleew17: boy dats cold...tyga dwn ba...  \n",
       "2  !!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...  \n",
       "3  !!!!!!!!! RT @C_G_Anderson: @viva_based she lo...  \n",
       "4  !!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_davidson = pd.read_csv(\"hate_speech_labelled_data/labeled_data.csv\")\n",
    "X_train_davidson.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qXu00UY5V1ku"
   },
   "source": [
    "### Columns key:\n",
    "\n",
    "-   count = number of CrowdFlower users who coded each tweet (min is 3, sometimes more users coded a tweet when judgments were determined to be unreliable by CF).\n",
    "\n",
    "\n",
    "-   hate_speech = number of CF users who judged the tweet to be hate speech.\n",
    "\n",
    "\n",
    "-   offensive_language = number of CF users who judged the tweet to be offensive.\n",
    "\n",
    "\n",
    "-   neither = number of CF users who judged the tweet to be neither offensive nor non-offensive.\n",
    "\n",
    "\n",
    "-   class = class label for majority of CF users.\n",
    "\n",
    "    -   0 - hate speech\n",
    "    -   1 - offensive  language\n",
    "    -   2 - neither\n",
    "\n",
    "-   tweet = raw tweet text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 285
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1149,
     "status": "ok",
     "timestamp": 1588673163504,
     "user": {
      "displayName": "Samridha Man Shrestha",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjWM1giZm-4hM3_aud8FxlOUc8VWuoYWktFbs7VDA=s64",
      "userId": "14722659845832153120"
     },
     "user_tz": -240
    },
    "id": "hV-qiF1GV1kv",
    "outputId": "c5463840-0ee0-4ba1-fda4-97e8799bcefd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x127cc3b50>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD7CAYAAACIYvgKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAWiklEQVR4nO3df7BfdZ3f8eerQVjXH0uQu2k2ISbYuDvg7Ea4g9RVyy4rBOwabDs2zFaiS41U6Oh0p924zhRHyxS769phanFQM4YZC7KiktZQjBHXaW2QoDEQFHOJUJKJSZa4onWHXfDdP76fq4fLvbnfe7/3fhPJ8zFz5nu+7/M553y+536T1z3nc77fm6pCknRi+3vHugOSpGPPMJAkGQaSJMNAkoRhIEnCMJAk0UcYJDkjyd1JHkyyO8m7Wv20JFuT7GmPC1s9SW5IMpZkV5JzOtta19rvSbKuUz83yf1tnRuSZD5erCRpcv2cGTwF/FFVnQWcD1yd5CxgA7CtqlYC29pzgEuAlW1aD9wIvfAArgVeBZwHXDseIK3N2zvrrR78pUmS+nXSdA2q6gBwoM3/KMm3gSXAGuCC1mwT8BXgj1v95up9mm17klOTLG5tt1bVEYAkW4HVSb4CvLiqtrf6zcBlwJ1H69fpp59ey5cvn8FLlSTdd999f1VVIxPr04ZBV5LlwCuBe4BFLSgAvg8savNLgMc6q+1rtaPV901SP6rly5ezY8eOmXRfkk54SR6drN73AHKSFwK3A++uqie6y9pZwLx/r0WS9Ul2JNlx+PDh+d6dJJ0w+gqDJM+jFwSfqqrPtvLBdvmH9nio1fcDZ3RWX9pqR6svnaT+LFV1U1WNVtXoyMizznIkSbPUz91EAT4BfLuq/ryzaDMwfkfQOuCOTv2KdlfR+cAP2+Wku4CLkixsA8cXAXe1ZU8kOb/t64rOtiRJQ9DPmMFvA28B7k+ys9X+BLgeuC3JlcCjwJvbsi3ApcAY8BPgbQBVdSTJB4B7W7v3jw8mA+8EPgk8n97A8VEHjyVJcyu/qF9hPTo6Wg4gS9LMJLmvqkYn1v0EsiTJMJAkGQaSJAwDSRIz/ASypOkt3/CFY7bvR65/wzHbt36xeWYgSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEmijzBIsjHJoSQPdGqfTrKzTY+M/23kJMuT/E1n2Uc765yb5P4kY0luSJJWPy3J1iR72uPC+XihkqSp9XNm8ElgdbdQVf+8qlZV1SrgduCzncUPjy+rqqs69RuBtwMr2zS+zQ3AtqpaCWxrzyVJQzRtGFTVV4Ejky1rv92/GbjlaNtIshh4cVVtr6oCbgYua4vXAJva/KZOXZI0JIOOGbwWOFhVezq1FUm+meQvk7y21ZYA+zpt9rUawKKqOtDmvw8sGrBPkqQZGvQvnV3OM88KDgDLqurxJOcCn09ydr8bq6pKUlMtT7IeWA+wbNmyWXZZkjTRrM8MkpwE/BPg0+O1qnqyqh5v8/cBDwMvB/YDSzurL201gIPtMtL45aRDU+2zqm6qqtGqGh0ZGZlt1yVJEwxymej3gO9U1c8u/yQZSbKgzZ9Jb6B4b7sM9ESS89s4wxXAHW21zcC6Nr+uU5ckDUk/t5beAvwf4NeT7EtyZVu0lmcPHL8O2NVuNf0McFVVjQ8+vxP4ODBG74zhzla/Hnh9kj30Aub6AV6PJGkWph0zqKrLp6i/dZLa7fRuNZ2s/Q7gFZPUHwcunK4fkqT54yeQJUmGgSTJMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSaK/v4G8McmhJA90au9Lsj/JzjZd2ln2niRjSR5KcnGnvrrVxpJs6NRXJLmn1T+d5OS5fIGSpOn1c2bwSWD1JPUPV9WqNm0BSHIWsBY4u63zX5MsSLIA+AhwCXAWcHlrC/DBtq1/APwAuHKQFyRJmrlpw6Cqvgoc6XN7a4Bbq+rJqvoeMAac16axqtpbVX8L3AqsSRLgd4HPtPU3AZfN8DVIkgY0yJjBNUl2tctIC1ttCfBYp82+Vpuq/hLgr6vqqQl1SdIQzTYMbgReBqwCDgAfmrMeHUWS9Ul2JNlx+PDhYexSkk4IswqDqjpYVU9X1U+Bj9G7DASwHzij03Rpq01Vfxw4NclJE+pT7femqhqtqtGRkZHZdF2SNIlZhUGSxZ2nbwLG7zTaDKxNckqSFcBK4OvAvcDKdufQyfQGmTdXVQF3A/+srb8OuGM2fZIkzd5J0zVIcgtwAXB6kn3AtcAFSVYBBTwCvAOgqnYnuQ14EHgKuLqqnm7buQa4C1gAbKyq3W0XfwzcmuQ/AN8EPjFnr06S1Jdpw6CqLp+kPOV/2FV1HXDdJPUtwJZJ6nv5+WUmSdIx4CeQJUmGgSTJMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSaKPMEiyMcmhJA90an+a5DtJdiX5XJJTW315kr9JsrNNH+2sc26S+5OMJbkhSVr9tCRbk+xpjwvn44VKkqbWz5nBJ4HVE2pbgVdU1W8C3wXe01n2cFWtatNVnfqNwNuBlW0a3+YGYFtVrQS2teeSpCGaNgyq6qvAkQm1L1bVU+3pdmDp0baRZDHw4qraXlUF3Axc1havATa1+U2duiRpSOZizOAPgTs7z1ck+WaSv0zy2lZbAuzrtNnXagCLqupAm/8+sGgO+iRJmoGTBlk5yXuBp4BPtdIBYFlVPZ7kXODzSc7ud3tVVUnqKPtbD6wHWLZs2ew7Lkl6hlmfGSR5K/CPgT9ol36oqier6vE2fx/wMPByYD/PvJS0tNUADrbLSOOXkw5Ntc+quqmqRqtqdGRkZLZdlyRNMKswSLIa+HfAG6vqJ536SJIFbf5MegPFe9tloCeSnN/uIroCuKOtthlY1+bXdeqSpCGZ9jJRkluAC4DTk+wDrqV399ApwNZ2h+j2dufQ64D3J/k74KfAVVU1Pvj8Tnp3Jj2f3hjD+DjD9cBtSa4EHgXePCevTJLUt2nDoKoun6T8iSna3g7cPsWyHcArJqk/Dlw4XT8kSfPHTyBLkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CSRJ9hkGRjkkNJHujUTkuyNcme9riw1ZPkhiRjSXYlOaezzrrWfk+SdZ36uUnub+vckPaHlSVJw9HvmcEngdUTahuAbVW1EtjWngNcAqxs03rgRuiFB3At8CrgPODa8QBpbd7eWW/iviRJ86ivMKiqrwJHJpTXAJva/Cbgsk795urZDpyaZDFwMbC1qo5U1Q+ArcDqtuzFVbW9qgq4ubMtSdIQDDJmsKiqDrT57wOL2vwS4LFOu32tdrT6vknqkqQhmZMB5PYbfc3Fto4myfokO5LsOHz48HzvTpJOGIOEwcF2iYf2eKjV9wNndNotbbWj1ZdOUn+WqrqpqkaranRkZGSArkuSugYJg83A+B1B64A7OvUr2l1F5wM/bJeT7gIuSrKwDRxfBNzVlj2R5Px2F9EVnW1JkobgpH4aJbkFuAA4Pck+encFXQ/cluRK4FHgza35FuBSYAz4CfA2gKo6kuQDwL2t3furanxQ+p307lh6PnBnmyRJQ9JXGFTV5VMsunCStgVcPcV2NgIbJ6nvAF7RT18kSXPPTyBLkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CSxABhkOTXk+zsTE8keXeS9yXZ36lf2lnnPUnGkjyU5OJOfXWrjSXZMOiLkiTNTF9/A3kyVfUQsAogyQJgP/A54G3Ah6vqz7rtk5wFrAXOBn4N+FKSl7fFHwFeD+wD7k2yuaoenG3fJEkzM+swmOBC4OGqejTJVG3WALdW1ZPA95KMAee1ZWNVtRcgya2trWEgSUMyV2MGa4FbOs+vSbIrycYkC1ttCfBYp82+VpuqLkkakoHDIMnJwBuBv2ilG4GX0buEdAD40KD76OxrfZIdSXYcPnx4rjYrSSe8uTgzuAT4RlUdBKiqg1X1dFX9FPgYP78UtB84o7Pe0labqv4sVXVTVY1W1ejIyMgcdF2SBHMTBpfTuUSUZHFn2ZuAB9r8ZmBtklOSrABWAl8H7gVWJlnRzjLWtraSpCEZaAA5yQvo3QX0jk75PyVZBRTwyPiyqtqd5DZ6A8NPAVdX1dNtO9cAdwELgI1VtXuQfkmSZmagMKiq/we8ZELtLUdpfx1w3ST1LcCWQfoiSZo9P4EsSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEnMQBkkeSXJ/kp1JdrTaaUm2JtnTHhe2epLckGQsya4k53S2s66135Nk3aD9kiT1b67ODH6nqlZV1Wh7vgHYVlUrgW3tOcAlwMo2rQduhF54ANcCrwLOA64dDxBJ0vybr8tEa4BNbX4TcFmnfnP1bAdOTbIYuBjYWlVHquoHwFZg9Tz1TZI0wVyEQQFfTHJfkvWttqiqDrT57wOL2vwS4LHOuvtabaq6JGkITpqDbbymqvYn+VVga5LvdBdWVSWpOdgPLWzWAyxbtmwuNilJYg7ODKpqf3s8BHyO3jX/g+3yD+3xUGu+Hzijs/rSVpuqPnFfN1XVaFWNjoyMDNp1SVIzUBgkeUGSF43PAxcBDwCbgfE7gtYBd7T5zcAV7a6i84EftstJdwEXJVnYBo4vajVJ0hAMeploEfC5JOPb+m9V9T+T3AvcluRK4FHgza39FuBSYAz4CfA2gKo6kuQDwL2t3fur6siAfZMk9WmgMKiqvcBvTVJ/HLhwknoBV0+xrY3AxkH6I0maHT+BLEkyDCRJhoEkCcNAksTcfOhMkk44yzd84Zjs95Hr3zAv2/XMQJJkGEiSDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CSxABhkOSMJHcneTDJ7iTvavX3JdmfZGebLu2s854kY0keSnJxp7661caSbBjsJUmSZmqQr7B+CvijqvpGkhcB9yXZ2pZ9uKr+rNs4yVnAWuBs4NeALyV5eVv8EeD1wD7g3iSbq+rBAfomSZqBWYdBVR0ADrT5HyX5NrDkKKusAW6tqieB7yUZA85ry8aqai9AkltbW8NAkoZkTsYMkiwHXgnc00rXJNmVZGOSha22BHiss9q+VpuqLkkakoHDIMkLgduBd1fVE8CNwMuAVfTOHD406D46+1qfZEeSHYcPH56rzUrSCW+gMEjyPHpB8Kmq+ixAVR2sqqer6qfAx/j5paD9wBmd1Ze22lT1Z6mqm6pqtKpGR0ZGBum6JKljkLuJAnwC+HZV/XmnvrjT7E3AA21+M7A2ySlJVgArga8D9wIrk6xIcjK9QebNs+2XJGnmBrmb6LeBtwD3J9nZan8CXJ5kFVDAI8A7AKpqd5Lb6A0MPwVcXVVPAyS5BrgLWABsrKrdA/RLkjRDg9xN9L+ATLJoy1HWuQ64bpL6lqOtJ0maX34CWZJkGEiSDANJEoaBJAnDQJKEYSBJYrDPGfzCWr7hC8dkv49c/4Zjsl9Jmo5nBpIkw0CSZBhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJI4jsIgyeokDyUZS7LhWPdHkk4kx0UYJFkAfAS4BDgLuDzJWce2V5J04jguwgA4Dxirqr1V9bfArcCaY9wnSTphHC9hsAR4rPN8X6tJkobgF+rvGSRZD6xvT3+c5KFZbup04K/mplf9ywenbXJM+tUH+zUzx6xf07zHPF4zc1z2Kx8cuF8vnax4vITBfuCMzvOlrfYMVXUTcNOgO0uyo6pGB93OXLNfM2O/ZsZ+zcyJ1q/j5TLRvcDKJCuSnAysBTYf4z5J0gnjuDgzqKqnklwD3AUsADZW1e5j3C1JOmEcF2EAUFVbgC1D2t3Al5rmif2aGfs1M/ZrZk6ofqWq5mO7kqRfIMfLmIEk6Rh6zoXBdF9rkeSUJJ9uy+9Jsryz7D2t/lCSi4fcr3+T5MEku5JsS/LSzrKnk+xs05wOrPfRr7cmOdzZ/7/sLFuXZE+b1g25Xx/u9Om7Sf66s2xejleSjUkOJXlgiuVJckPr864k53SWzeexmq5ff9D6c3+SryX5rc6yR1p9Z5IdQ+7XBUl+2PlZ/fvOsnn7epo++vVvO316oL2fTmvL5vN4nZHk7vb/wO4k75qkzfy9x6rqOTPRG3x+GDgTOBn4FnDWhDbvBD7a5tcCn27zZ7X2pwAr2nYWDLFfvwP8cpv/V+P9as9/fAyP11uB/zLJuqcBe9vjwja/cFj9mtD+X9O76WC+j9frgHOAB6ZYfilwJxDgfOCe+T5Wffbr1eP7o/eVL/d0lj0CnH6MjtcFwP8Y9Oc/1/2a0Pb3gS8P6XgtBs5p8y8CvjvJv8d5e489184M+vlaizXApjb/GeDCJGn1W6vqyar6HjDWtjeUflXV3VX1k/Z0O73PWsy3Qb4G5GJga1UdqaofAFuB1ceoX5cDt8zRvqdUVV8FjhylyRrg5urZDpyaZDHze6ym7VdVfa3tF4b33urneE1lXr+eZob9Gsp7C6CqDlTVN9r8j4Bv8+xvYpi399hzLQz6+VqLn7WpqqeAHwIv6XPd+exX15X00n/cLyXZkWR7ksvmqE8z6dc/baekn0ky/uHA4+J4tctpK4Avd8rzdbymM1W/j6evW5n43irgi0nuS+8T/sP2D5N8K8mdSc5utePieCX5ZXr/od7eKQ/leKV3+fqVwD0TFs3be+y4ubVUPUn+BTAK/KNO+aVVtT/JmcCXk9xfVQ8PqUv/Hbilqp5M8g56Z1W/O6R992Mt8JmqerpTO5bH67iV5HfohcFrOuXXtGP1q8DWJN9pvzkPwzfo/ax+nORS4PPAyiHtux+/D/zvquqeRcz78UryQnoB9O6qemIut300z7Uzg36+1uJnbZKcBPwK8Hif685nv0jye8B7gTdW1ZPj9ara3x73Al+h9xvDUPpVVY93+vJx4Nx+153PfnWsZcJp/Dwer+lM1e/5PFZ9SfKb9H5+a6rq8fF651gdAj7H3F0anVZVPVFVP27zW4DnJTmd4+B4NUd7b83L8UryPHpB8Kmq+uwkTebvPTYfAyHHaqJ3prOX3mWD8YGnsye0uZpnDiDf1ubP5pkDyHuZuwHkfvr1SnqDZisn1BcCp7T504E9zNFgWp/9WtyZfxOwvX4+YPW91r+Fbf60YfWrtfsNegN6GcbxattcztQDom/gmYN7X5/vY9Vnv5bRGwN79YT6C4AXdea/BqweYr/+/vjPjt5/qv+3Hbu+fv7z1a+2/FfojSu8YFjHq732m4H/fJQ28/Yem7ODe7xM9Ebbv0vvP9b3ttr76f22DfBLwF+0fxxfB87srPvett5DwCVD7teXgIPAzjZtbvVXA/e3fxD3A1cOuV//Edjd9n838Buddf+wHccx4G3D7Fd7/j7g+gnrzdvxovdb4gHg7+hdk70SuAq4qi0PvT/S9HDb9+iQjtV0/fo48IPOe2tHq5/ZjtO32s/4vUPu1zWd99Z2OmE12c9/WP1qbd5K74aS7nrzfbxeQ29MYlfnZ3XpsN5jfgJZkvScGzOQJM2CYSBJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJKA/w/DqQlXF0MlkwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_train_davidson['class'].hist(grid=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uc1mK2lTV1ky"
   },
   "source": [
    "This histogram shows the imbalanced nature of the task - most tweets containing \"hate\" words as defined by Hatebase were \n",
    "only considered to be offensive by the CF coders. More tweets were considered to be neither hate speech nor offensive language than were considered hate speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 190,
     "referenced_widgets": [
      "48645f6e13b641b7a57276eabeb12eb4",
      "e24df47fb21d443f9bc34844ac5fa682",
      "859bab5cb1784f4095ad1c8b532ce68c",
      "25991d8eb42349c398dbb6391ece0e5d",
      "f14e68c945e34132a3bddfa0452d7a16",
      "44777498fa564a2695f6334f000bf9bf",
      "d022d2a3196b406291a9c366d511761a",
      "7c7575a3bbac408aafca0461affc5467"
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 38158,
     "status": "ok",
     "timestamp": 1588673227378,
     "user": {
      "displayName": "Samridha Man Shrestha",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjWM1giZm-4hM3_aud8FxlOUc8VWuoYWktFbs7VDA=s64",
      "userId": "14722659845832153120"
     },
     "user_tz": -240
    },
    "id": "NQaUh-vByzAP",
    "outputId": "a767bd0f-f62a-4257-aa86-bebd6bc8cefd"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sam/Downloads/GitHub/covid_19_hate_speech/venv/lib/python3.7/site-packages/sklearn/feature_extraction/text.py:386: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['b', 'c', 'e', 'f', 'g', 'h', 'j', 'l', 'n', 'p', 'r', 'u', 'v', 'w'] not in stop_words.\n",
      "  'stop_words.' % sorted(inconsistent))\n",
      "100%|██████████| 24783/24783 [01:18<00:00, 315.61it/s]\n"
     ]
    }
   ],
   "source": [
    "X_train_davidson_tweets = X_train_davidson['tweet']\n",
    "M, vectorizer, pos_vectorizer = generate_features_train_data(X_train_davidson_tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IcOM_yE1V1la"
   },
   "source": [
    "# Training the model and testing it\n",
    "\n",
    "The best model was selected using a GridSearch with 5-fold CV."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XHpELp9eV1lb"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold, GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "X = pd.DataFrame(M)\n",
    "y = X_train_davidson['class'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lDr2ljmkV1lm"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 72
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1788,
     "status": "ok",
     "timestamp": 1588673229189,
     "user": {
      "displayName": "Samridha Man Shrestha",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjWM1giZm-4hM3_aud8FxlOUc8VWuoYWktFbs7VDA=s64",
      "userId": "14722659845832153120"
     },
     "user_tz": -240
    },
    "id": "XjqPTAksV1lq",
    "outputId": "7ba0bba8-d421-4567-e098-1de62e862a1d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sam/Downloads/GitHub/covid_19_hate_speech/venv/lib/python3.7/site-packages/sklearn/model_selection/_split.py:297: FutureWarning: Setting a random_state has no effect since shuffle is False. This will raise an error in 0.24. You should leave random_state to its default (None), or set shuffle=True.\n",
      "  FutureWarning\n"
     ]
    }
   ],
   "source": [
    "pipe = Pipeline(\n",
    "        [('select', SelectFromModel(LogisticRegression(class_weight='balanced',\n",
    "                                                       penalty=\"l1\", \n",
    "                                                       C=0.01, \n",
    "                                                       solver='liblinear'))),\n",
    "        ('model', LogisticRegression(class_weight='balanced', penalty='l2'))])\n",
    "\n",
    "param_grid = [{}] # Optionally add parameters here\n",
    "grid_search = GridSearchCV(pipe, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, random_state=42).split(X_train, y_train), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YSbtZC0qV1ly"
   },
   "source": [
    "# Train the Model and optimize with grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 30910,
     "status": "ok",
     "timestamp": 1588673258319,
     "user": {
      "displayName": "Samridha Man Shrestha",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjWM1giZm-4hM3_aud8FxlOUc8VWuoYWktFbs7VDA=s64",
      "userId": "14722659845832153120"
     },
     "user_tz": -240
    },
    "id": "--n6_499V1l1",
    "outputId": "a6c61868-18cb-4d37-f501-567ef845c308",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sam/Downloads/GitHub/covid_19_hate_speech/venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    5.9s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................................................. , total=   5.9s\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sam/Downloads/GitHub/covid_19_hate_speech/venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................................................. , total=   7.7s\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sam/Downloads/GitHub/covid_19_hate_speech/venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................................................. , total=   7.6s\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sam/Downloads/GitHub/covid_19_hate_speech/venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................................................. , total=  10.7s\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sam/Downloads/GitHub/covid_19_hate_speech/venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   38.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................................................. , total=   6.3s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sam/Downloads/GitHub/covid_19_hate_speech/venv/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    }
   ],
   "source": [
    "model = grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zGBIhO9cV1l7"
   },
   "source": [
    "## Evaluating the results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "M-3A56uPV1l_"
   },
   "source": [
    "***Note: Results in paper are from best model retrained on the entire dataset (see the other notebook). Here the results are reported after using cross-validation and only for the held-out set.***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 190
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 17708,
     "status": "ok",
     "timestamp": 1588673258320,
     "user": {
      "displayName": "Samridha Man Shrestha",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjWM1giZm-4hM3_aud8FxlOUc8VWuoYWktFbs7VDA=s64",
      "userId": "14722659845832153120"
     },
     "user_tz": -240
    },
    "id": "c-D4ZviCV1mA",
    "outputId": "0420afa9-4bc5-43c3-8313-58a188c16467"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.30      0.70      0.42       164\n",
      "           1       0.97      0.82      0.89      1905\n",
      "           2       0.76      0.92      0.83       410\n",
      "\n",
      "    accuracy                           0.83      2479\n",
      "   macro avg       0.68      0.81      0.71      2479\n",
      "weighted avg       0.90      0.83      0.85      2479\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_preds = model.predict(X_test)\n",
    "report = classification_report(y_test, y_preds)\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 341
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1320,
     "status": "ok",
     "timestamp": 1588673259650,
     "user": {
      "displayName": "Samridha Man Shrestha",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjWM1giZm-4hM3_aud8FxlOUc8VWuoYWktFbs7VDA=s64",
      "userId": "14722659845832153120"
     },
     "user_tz": -240
    },
    "id": "5qBv-C_EV1mE",
    "outputId": "c5614528-5765-4dfe-9a6f-6d0e8b47f184"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUQAAAFECAYAAABf6kfGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU1f3/8dcHEiEJm4oiIJsLglgWixugAi6ARQVbvmot1qXurYp79WtVlNat1oK4K9Sd/vzivqIgWnEDAVdQQBBB0KICWQgk+fz+ODdxsk8guZPA+/l4zCMz956599ybmc+cc8+555i7IyIi0CjVGRARqS8UEEVEIgqIIiIRBUQRkYgCoohIRAFRRCSSluoMbI5evXqpr1AMZsyYkeosbDNWr16d6ixsM7p3726VrVMJUUQkooAoIhJRQBQRiSggiohEFBBFRCIKiCIiEQVEEZGIAqKISEQBUUQkooAoIhJRQBQRiSggiohEFBBFRCIKiCIiEQVEEZGIAqKISEQBUUQkooAoIhJRQBQRiSggiohEFBBFRCIKiCIiEQVEEZGIAqKISEQBUUQkooAoIhJRQBQRiSggiohEFBBFRCIKiCIiEQVEEZGIAqKISEQBUUQkooAoIhJRQBQRiSggiohEFBBFRCIKiCIiEQVEEZFIWqoz0JC1aNGC6667joMOOogff/yR8ePH89JLL1WYtlu3blx22WV0796dvLw87r//fh577DEAevXqxWWXXUaXLl1YsWIFf/3rX5k7d26ch1LvrV27lr/+9a+8//77tGrVirPPPpshQ4aUSzdnzhwefPBBFi5cSPPmzXnqqadK1v3www/cfvvtzJ07l7y8PHbbbTcuuOACevToEeeh1Gvr16/njjvuYN68ebRo0YLf/e53HHrooeXSffzxx0yZMoUlS5aQlZXFfffdV2r9ggULeOCBB1i+fDlt2rThrLPOYu+9947rMDabSohb4Morr2TTpk0MGjSIK6+8kquuuordd9+9XLpWrVpx11138eSTT3LIIYcwfPhw3nnnHSAE1fHjxzN58mQGDBjA5MmTGT9+PM2bN4/7cOq1v//976Snp/PCCy9w7bXXcsstt7BkyZJy6Zo2bcrw4cP54x//WG5dXl4e3bt3Z9KkSbzyyiscddRRXHzxxeTm5sZxCA3CvffeS1paGpMnT2bMmDHcc889fP311+XSNWnShMMOO4zf//735datX7+ecePGMWLECB599FFGjhzJuHHjyM7OjuMQtogC4mbKyMjg8MMPZ+LEieTl5TF37lxmzpzJ8OHDy6UdPXo0s2bN4sUXX2TTpk3k5uby1VdfAdC7d2/WrFnDtGnTKCoq4oUXXuDHH3/k8MMPj/uQ6q28vDxmzJjBmWeeSWZmJr169eLggw/m5ZdfLpe2R48eDBs2jHbt2pVb1759e0488URat25N48aNGTFiBJs2barwC78t2rBhA++88w6//e1vycjIYO+992a//fbjjTfeKJe2a9euDBo0iF122aXcugULFtCqVSv69+9P48aNGThwIC1btiwpBNRnCoibqVOnThQUFLBs2bKSZQsXLqywhNizZ0/Wrl3Lv/71L2bMmMH48eMr/CAlqmg726qvv/6axo0b07Fjx5Jle+yxR4UlxJr44osvKCgoYNddd93SLG4VVq5cSaNGjWjfvn3Jsi5dumzWD4a7l3vdEH54Yg+IZtbEzMaZ2RIzWxstO9LMytdx6rGMjAxycnJKLcvOziYzM7Nc2jZt2nD00Udz8803M2TIEFasWMGNN94IwPz589lpp50YOnQoaWlpHH300XTo0IGMjIxYjqMhyMvLIysrq9SyZs2abVFVNycnh+uuu47TTjuNZs2abWkWtwp5eXnlPr+ZmZnk5eXVaDt77bUXP/zwA2+++SYFBQVMnz6dVatWkZ+fX5vZrROpKCH+A9gHOAko/hn5FDinqjeZ2ZlmNtvMZq9Zs6aOs1i9mnxJN2zYwIwZM/j000/ZuHEjd999N3369KFZs2asXbuWCy+8kNGjRzN9+nT69+/Pe++9x+rVq+M6lHqvoh+fnJycCn98krFhwwYuueQS9tlnnwqvgW2rMjIyyn1+c3Nza/zj3KJFC6688kqeffZZTjnlFObOnUvPnj3ZcccdazO7dSIVrcwjgT3cPcfMigDcfYWZta/qTe5+L3AvQK9evbyqtHFYtmwZaWlpdOzYsaQq0LVrVxYvXlwu7ZdfflmqClG2OjFnzhxOOukkABo3bswLL7zAQw89VIe5b1g6duxIYWEhy5cvp0OHDkA4p7vttluNt7Vx40auuOIKdt55Zy6//PLazmqD1q5dO4qKili5cmXJNdilS5eWulSRrH322Ydbb70VgMLCQs466yxGjBhRq/mtC6koIW6kTCA2s52A1Bf7aiAvL4/XX3+dc889l4yMDHr37s3AgQN5/vnny6V95plnGDx4MHvttRdpaWmceeaZfPjhhyWtbt26dSMtLY2srCwuuugiVq1axaxZs+I+pHorIyODgQMHct9995GXl8f8+fN56623GDp0aLm0RUVF5OfnU1hYCEB+fj6bNm0CoKCggCuvvJImTZpw9dVX06iRLqEnatq0KQceeCCPP/44GzZs4PPPP+f9999n4MCB5dIWFRWxceNGCgoKgPBDU3yeAZYsWUJBQQG5ublMmjSJ1q1b06dPn7gOZbNZ2dJKne/Q7FZgD2AMMAfoAdwOLHL3q5LZRn0oIULpfog//fQT//znP3nppZfo06cPd955JwcddFBJ2lGjRnHmmWfStGlT5s6dy7hx40qqxTfeeCMDBgwAYNasWdx444388MMPKTmmRDNmzEh1Fkok9kNs2bIl55xzDkOGDGHevHlcdNFFTJ8+HYAPP/yQ8847r9R7i/8fxeuaNGlSKhjedttt9O7dO9bjKau+XCJZv349EyZMYP78+TRv3pzRo0dz6KGH8umnn3L99dfzxBNPAKEf4tVXX13qvT169GDcuHFA6CY1Z84cIJz/M844g1atWsV7MJXo3r27VbYuFQFxO+Am4AwgE8gF7gOucPekrrrWl4C4tatPAXFrV18C4ragqoAY+zVEd99IKB2OiarK//W4o7KISAVS0e2mpC7o7t8XB0Mz+y7uvIiIJErFVeX0sgvMLB1onIK8iIiUiK3KbGZvEfodNjWzN8us3hVQs6qIpFSc1xDvBwzYD3ggYbkDq4HpMeZFRKSc2AKiu/8LwMzedfcFce1XRCRZqWhlXmBmbYD9gdaEUmPxugfjzo+ISLHYA6KZjQAeAb4kdMr+lHBv838ABUQRSZlUtDLfAJzq7n2AnOjvmYS7VkREUiYVAbGju/+/Msv+BZycgryIiJRIRUD8LrqGCLDUzA4Cdkf9EEUkxVIREO8DBkTP/wHMAOYDd6UgLyIiJVLRynxTwvOHzOwNIMvdP487LyIiiVJxp0pl63H3Q+LKj4hIWXHfqVLMgInAuTHuX0SkSrHfqVLMzG4ru0xEJJU0hrqISEQBUUQkEmejyuCy+zazQZS+l1kj3ohIysTZqPJAmddrKH3vsgM1n1dSRKSWxNmo0iWufYmIbA5dQxQRiSggiohEFBBFRCIKiCIiEQVEEZGIAqKISEQBUUQkooAoIhJRQBQRiSggiohEFBBFRCIKiCIikc0KiGaWYWaHm1mn2s6QiEiqJBUQzWyymZ0bPd8OeB94FVhoZsPqMH8iIrFJtoQ4BHg3en4M0BzYBbg2eoiINHjJBsTtge+i50OB/3P374AngL3rImMiInFLNiCuAvYxs8aE0uJr0fJmwKa6yJiISNySHTH7QWAKsBIoBF6Plh8ALKiDfImIxC6pgOjuY83sU6Aj8P/cfWO0qgC4qa4yJyISJ3P3VOehxlauXNnwMt0ADRw4MNVZ2GbMnDkz1VnYZrRt29YqW5d0P0QzG2Zmz5vZZ2bWIVr2BzM7rDYyKSKSasn2QzwJ+DfwJdAFSI9WNQYuq5usiYjEK9kS4mXAGe4+hnDdsNi7QO9az5WISAokGxD3BN6pYHk20KL2siMikjrJBsSVQNcKlh8CLK697IiIpE6yAfFeYLyZ9Y9edzCz3wM3A3fVSc5ERGKWbD/Em82sJTANaArMAPKBW919Yh3mT0QkNsneqYK7X2Vm4wj3LjcCPnP37DrLmYhIzJIOiADungvMrqO8iIikVKUB0cyeBX7n7uui55Vy92NqPWciIjGrqoS4Bii+Re6HhOciIlulSgOiu5+a8PJcIN/dC+s+SyIiqVFtt5toDMS1wF51nx0RkdSpNiBGpcJlwHZ1nx0RkdRJtmP29cCNZta6LjMjIpJKyXa7uYQwys0KM/sGyElc6e49aztjIiJxSzYgPlmnuRARqQeSvXXvurrOiIhIqtXoThUzG0y4dc+BT939jbrIlIhIKiQVEM2sPfAU8EvCUGAA7cxsNjDS3VdW+mYRkQYi2Vbm8YTpR/dw9w7u3oEwaGxhtE5EpMFLtsp8BDDQ3b8qXuDuS8zsfH6eo1lEpEFLetY9Kr6XWfc3i8hWI9mA+DowoXj6UQAz6wjcjkqIIrKVSDYgng9kAUvMbJmZLSPMpZIVrRMRafCS7Ye43Mz2BQ4HukWLP3f31+osZyIiMavJFAJOmFNlWt1lR0QkdZLth/iXSlY5sAFYBLzs7nm1lTERkbglW0IcBXQkXDMs6ZhNGOThe6AD8J2ZHeruS2o9lyIiMUi2UeXvwAdAZ3fv6O4dgc7Ae8BYQnD8AritLjIpIhKHZAPiNcBF7v5N8YLo+WXAWHdfA1wFHFT7WRQRiUeyAbENYYL6spoAO0fPVwOZtZEpEZFUSDYgvgbcY2b7mVmj6LEfcBc/tzr/Aviq0i2IiNRzyQbEPxBKgO8B+dHj3WjZGVGa9YSRtUVEGqRkO2Z/Bww1s734efa9Be7+RUKaGXWQPxGR2NRogFh3X2hmPwHfu3tRHeVJRCQlkqoym1m6md1sZuuBFYQuN5jZTWZ2bh3mT0QkNjXpdnM08DvC9cNi7wOn1HKeRERSItkq84nAae4+08wSq8qfAF1rP1siIvFLtoTYDlhWwfI0angdUkSkvko2IH4KHFLB8v8B5tRedhqWdevWcfXVVzNs2DBOOOEEXnut4tHQ5s6dy5gxYxg+fDgnnHBCpdubN28egwYN4oEHHqirLDdYLVu2ZOLEicybN48ZM2YwfPjwCtPdf//9zJ07t+TxySef8NxzzwGwww47cNttt/HWW28xZ84cHn/8cXr27BnnYdR769at43//938ZOnQoxx9/fJWf6QsvvJBf/epXHH/88ZVub968eQwcOJD777+/rrJcq5It3V0HPBKNmN0YGGVm3YDfAr+qq8zVd//85z9JS0tj6tSpLFq0iD//+c/svvvudOnSpVS6pk2bMmzYMAYPHsyjjz5a4bYKCgq444476N69exxZb3CuueYaNm3aRL9+/ejevTv33nsvCxYsYNGiRaXS/eEPfyj1+uGHH+bdd98FICsri48//pi//e1vrFmzhlGjRnHfffcxaNAgcnNzYzuW+uz2228nPT09qc/0UUcdRX5+Po888kiF2yooKGDChAkN6jOdVAnR3Z8jlAaPBIoIjSx7AkdvziCxZrajmY02s8ui1+3MbNeabieV8vLyePPNNznttNPIyMjgF7/4Bf369WPatPLDRXbv3p0jjzySdu3aVbq9f//73/Tt25eOHTvWZbYbpIyMDI488khuv/12cnNzmTNnDtOnT2fEiBFVvq99+/b07duXp59+GoDly5czadIkvv/+e4qKipgyZQrp6enlvuzbqsTPdGZmJj179qRfv368+uqr5dIWf6bbtm1b6famTJnCfvvt16A+00lPMuXur7j7oe7ezN0z3X2Au5c/U9Uws0OBhcBJwNXR4j0JtwE2GN988w2NGzemQ4eSaWbYfffdWbp0aY23tWrVKl566SV+//vf12IOtx6dO3emsLCw1Ln9/PPP2WOPPap834gRI5g9ezYrVqyocH337t1JT09n2bKKLo9ve+riM33yySfXYg7rXrL9EJeY2Y4VLG9lZjUd//B24Hh3HwoURMveA/av4XZSKi8vj8zM0mNZZGVlbVbVa8KECZx66qlkZGTUVva2KllZWWRnZ5dalp2dTVZWVpXvGzFiBFOnTq10mzfffDN33HFHuW1vqyr6TDdr1myzP9PFJc2GJNkSYmfCtcOymgDta7jPzu5ePFNf8TSmG6nmeqaZnWlms81sdmXXLOKUkZFR7oOSm5tb4w/ArFmzyMvLY/DgwbWZva1KTk4OzZo1K7WsWbNm5OTkVPqeX/7yl7Ru3ZpXXnml3LomTZpwzz33MH/+fO65555az29DVdFnOicnZ7M+07m5uQ3yM11dEDou4eWvzGxtwuvGwGHA0hru8zMzG+LuiZ/Uw4GPq3qTu98L3AuwcuXKlM8Hveuuu1JYWMg333zDrruGy5+LFi2ic+fONdrOhx9+yMKFCznuuHCqc3JyaNSoEUuWLGHcuHG1ne0GaenSpTRu3JhOnTqVVG+7detWrkEl0ciRI5k2bVq5L3h6ejp33nknq1at4uqrr67k3dumij7TixcvrvFnes6cOSxcuJCRI0cCP3+mv/rqq3r/ma6ulfnJ6K8DZfuCbCIEw4truM+LgefN7AUgw8zuIdwFc2wNt5NSGRkZHHzwwUyaNIlLLrmERYsWMWvWLCZMmFAubVFREQUFBRQUhCsEGzduxMxIT0/ntNNO47e//W1J2gkTJtC6dWtGjx4d27HUd3l5eUybNo0LLriAq666iu7du3PYYYdV2t2jSZMmDBs2jPPOO6/U8rS0NCZMmEB+fj6XX345Yd40KVb8mX7wwQe59NJLWbRoEW+//TZ33HFHubRFRUVs2rSJwsJCAPLz82nUqBHp6emcfvrppT7Td9xxBzvuuGODuJ5YZZXZ3Ru5eyPga2Dn4tfRo4m77+Xuz9dkh+7+LtCL0LfxQcIYivu7+webeQwpc+GFF5Kfn89xxx3HDTfcwIUXXkiXLl346KOPGDZsWEm6jz76iCFDhnDFFVewevVqhgwZwqWXXgpAZmYmO+ywQ8mjSZMmNG3alBYtWqTqsOqla6+9lqZNm/LOO+9w2223cc0117Bo0SL69u3L3LlzS6U9/PDDWbduXUl3m2J9+vRh8ODB9O/fn9mzZ5f0Vezbt2+ch1KvjRkzhvz8fEaOHMn111/PmDFjSj7TQ4cOLUk3f/58hgwZwuWXX17ymb7kkjD6X2ZmJjvuuGPJY7vttmswn2mL+1fSzHq7+7wt2UZ9qDJvCwYOHJjqLGwzZs6cmeosbDPatm1rla1L+rY7M9seGEaYfW+7xHXuPrYG+XnVzL4HHgcedXeNsi0i9UKy8zIfCLxAGOlmJ8IQYG2j10sJM+8lqy0wlDBgxHwz+xR4DJgSDUQrIpISyXa7uQV4lNDFZgMwmFBSnA3cVJMdunuhu7/g7r8jTF71T+A3wPKabEdEpLYlGxB7And4uOBYCDRx99XA5cC1m7NjM2sKDAeOB/oCb23OdkREakuyAXFjwvPVQKfoeTZhaLCkmdlRZvYI8B2hC85MYHd3P7wm2xERqW3JNqp8COwHfAG8AdxgZm0II2h/VMN93kpoUOnj7otr+F4RkTqTbEC8CmgePf9f4CFgAiFAnlaTHbr73jVJLyISl2SnIZ2d8Px7QvebpJnZVe4+LnpeaYu0u/+lJtsVEalNyXa76QE0dvePyizvCRS4+2fVbCJxrMMOlaYSEUmhZKvM9wITKX+9cG/gj8CAqt7s7uckPD+1JhkUEYlLsgGxJ2HK0bI+AH5Rkx2a2d7AGndfbWbNgEsJo3Df4u4ax11EUibZbjeFQMsKlm8PVHpfYCUeB1pFz28lTF51IKCB6UQkpZItIc4ErjKzUe5eCGBmaYTW5zdruM/O7r7QzAw4jlDtziOMeiMikjLJBsTLgP8Ai8zsP9GyAUAzKp6etCobzKw5IRB+7e7/jYJr0xpuR0SkViU7695CwnXEx4AdosejQC93/7yG+3wMmA78C5gcLdsXlRBFJMWSHv7L3b8lVJG3iLuPMbMjgU3uPiNaXASM2dJti4hsiaQDYm0qO31pYsdvEZFUiT0gmlkXYBzQm3ANsoS7N5wZrUVkq5OKEuJjwGLCSDfqdygi9UYqAmIPoL+7F6Vg3yIilUq2YzYAZtbazA4wsyZbsM83gT5b8H4RkTqR7OAOzQnzMv+GMEfznsASM7sbWOXu19Zgn0uBl83sKWBV4gqNdiMiqZRsCfEmwnwq+xLuKin2PDCyhvvMit6XThj5JvEhIpIyyV5DPAYY6e7zzCxxTuTPgd1qskONdiMi9VWyAXF7YE0Fy5sTBn6oETPrBowC2rj7H81sL8LEVTWdjkBEpNYkW2X+gFBKLFZcSjwLmFWTHZrZKMIMe+2Bk6PFzYHbarIdEZHalmwJ8UrglWjk7DTgouj5/tR8cIexwOHuPt/Mjo+WzQd61XA7IiK1KtnBHWYB/YDtCJ2qDwNWAge5+4c13OfO/Dzytif89YqTi4jEoyaDO3wM/L4W9jkHGE2Yua/YCVQ8IreISGyS7Ye4Q1Xr3f2HGuzzfOBVMzsdyDKzV4CuwJE12IaISK1LtoT4X6qu0jau6s1mtr27/wjg7guiVubhhP6Iy4Hn3T07ybyIiNSJZAPioDKv0wm3351DmLi+OsuAFgBm9pq7Hw78O9lMiojEIdmJ6mdWsPg1M1sC/IEwgk1Vcs1sH0JH7v2j+VTKTU6lAR9EJJW2dLSbeSTX7eY6QqNJ8aAQBWXWG6FKXmXVW0SkLm12QIzmVL6QcA2wSu5+l5ndB+wCLCAMASYiUq8k28q8ntKNKgZkAjnASUm8/113PxD4xsyecfdlm5NZEZG6lGwJ8Y9lXhcB3wPvFbceV6OrmTV19w3A0TXJoIhIXKoNiNGcyVnA0+6+cjP38wzwhZktBTLMrMLJ7d29prcBiojUmmoDorsXmNktwAubuxN3P9XMBgCdgf2A+6mglVlEJJWSrTK/C/yS0J9ws7j7f8zsXcL90EOA1oQO368DD7v7ps3dtohIbTD36sdUMLMTgL8C4wn3Iuckrk9mgAczawlMAzoBLwHfAm2BYcDXhBFw1iaT6fXr12sgiBg0b9481VnYZoSuuRIHd6/0ZFcZEM3sQULXmp+q3r5X23/QzO4kBMP/cfechOXNgCnAMnc/t7rtgAJiXBQQ46OAGJ8tCYiFhFJcRjU7qLYqbWYrgQPd/esK1nUG3nH3ttVtBxQQ46KAGB8FxPhUFRCru4Zo0QZqo99gS2BFJeu+IbrXWUQkVZIZILa2SmOLgcGVrDsMWFJL+xER2SzJBMRVZlZY1SPJfd0GPGRmvzazRgBm1sjMfgNMRnOqiEiKJdPt5kyqblRJirtPNrMdCcHvcTP7L6HrTT4w1t0nbek+RES2RHWNKkXALu7+Xa3t0Kw5YX6W4n6I77j7uppsQ40q8VCjSnzUqBKfLWlUqfXA4+7rgVdqe7siIluqumuI+tkSkW1GlSVEd092InsRkQZPAU9EJKKAKCISUUAUEYkoIIqIRBQQRUQiCogiIhEFRBGRiAKiiEhEAVFEJKKAKCISUUAUEYkoIIqIRBQQRUQiCogiIhEFRBGRiAKiiEhEAVFEJKKAKCISUUAUEYkoIIqIRBQQRUQiCogiIhEFRBGRiAKiiEhEAVFEJKKAKCISUUAUEYkoIIqIRBQQRUQiCohbYO3atVxyySUMGDCA4cOH8/LLL1eYbvbs2Zx11lkceuihHH300eXW33XXXRx//PEccMAB3HPPPXWd7Qbpp59+4rzzzqN3794MGjSI5557rsJ07s4tt9zCAQccwAEHHMAtt9yCuwPh/9CnT59Sj7322otXXnklzkOp17bffnumTp1KdnY2S5cu5cQTT6ww3Xbbbcddd93FqlWrWLNmDc8++yzt2rUrWf/www+zcuVK1q5dy8KFCzn99NPjOoQtooC4BW666SbS09N59dVXueGGG/jb3/7G4sWLy6Vr2rQpxxxzDBdccEGF2+nQoQPnn38+/fv3r+ssN1hjx44lPT2dt99+m1tuuYVrr72WL7/8sly6KVOm8Nprr/HMM8/w7LPPMmPGDJ544gkA+vbty9y5c0sed999N5mZmRx88MFxH069NXHiRDZu3EibNm046aSTuOuuu9h7773Lpbvgggs46KCD6NmzJ+3atePHH39kwoQJJev/9re/0blzZ1q2bMkxxxzDDTfcwL777hvnoWwWBcTNlJeXx/Tp0zn77LPJzMykd+/eHHLIIbz44ovl0u6zzz786le/on379hVua/jw4fTv35+srKy6znaDlJuby6uvvsoFF1xAVlYWffv2ZfDgwTzzzDPl0j799NOcdtpp7LLLLrRp04ZTTz2Vp556qsLtPv300wwdOpTMzMy6PoQGITMzk1//+tdcffXV5OTk8Pbbb/Pss88yevTocmm7dOnCK6+8wnfffUd+fj5TpkyhR48eJes/++wzNm7cCIRSu7uz++67x3YsmyvWgGhmjc3sNDNrEud+68KyZcto3LgxnTp1KlnWtWtXlixZksJcbZ2WLl1K48aN6dKlS8mybt26sWjRonJpv/zyS7p161YqXUUlydzcXF5++WVGjBhRN5lugLp27UpBQUGp8zV//vxSga7YAw88QP/+/Wnbti0ZGRmcdNJJvPTSS6XSTJw4kZycHBYuXMi3335bYWGhvok1ILp7IXCbu+fHud+6kJeXR7NmzUota9asGTk5OSnK0dYrNze33Llu3rx5hee6bNrmzZuTm5tbch2x2Kuvvsr222/P/vvvXzeZboCaNWvGunXrSi1bu3YtzZs3L5f2yy+/ZPny5axcuZJ169bRvXt3xo4dWyrNeeedR/PmzRkwYABTp04lP7/+f+1TUWV+zszKtyw0MBkZGWRnZ5dalpOTo2pvHcjMzCx3rrOzsys815mZmaUCZXZ2NpmZmZhZqXRPP/00I0aMKLd8W5adnU2LFi1KLWvRogXr168vl3bixIk0adKEHXbYgaysLKZOnVquhAhQVFTE29RBpi4AABL+SURBVG+/za677so555xTZ3mvLakIiE2BJ83sDTN72MweKn5U9SYzO9PMZpvZ7EmTJsWU1cp16tSJwsJCvv7665JlX3zxBbvttlsKc7V16ty5M4WFhSxdurRk2YIFC9hjjz3Kpd1zzz1ZsGBBqXR77rlnqTTffvst77//vqrLZXzxxRekpaWVOq+9evXi008/LZe2d+/eTJ48mR9//JGNGzcyYcIEDjjgAHbccccKt52WlqZriJX4BPgrMANYBCxOeFTK3e91977u3vfUU0+t+1xWIyMjg0GDBnH33XeTl5fHvHnzmDlzJkcddVS5tEVFReTn51NQUIC7k5+fz6ZNm0rWFxQUkJ+fT1FREYWFheTn51NYWBjn4dRrmZmZHHHEEYwfP57c3FzmzJnD66+/zrHHHlsu7bHHHsukSZNYvXo1q1evZtKkSYwcObJUmmeeeYY+ffrQsWPHuA6hQcjNzWXq1KmMHTuWzMxM+vXrx7HHHsvDDz9cLu0HH3zAySefTIsWLUhLS+Pcc89lxYoVrFmzhp122onjjz+erKwsGjVqxJFHHsmJJ57I66+/noKjqhkre22lIVi/fn29yPTatWsZO3Ys7733Hi1btuRPf/oTQ4cOZe7cuZx//vm89dZbQOj/dvbZZ5d677777su9994LwLXXXsvzzz9fav0111xTYZ/FOFV07ShVfvrpJ6688kpmzZpFq1atuPjiizn66KOZPXs2Z5xxBnPnzgV+7of45JNPAvCb3/yGSy+9tFTVeOjQoZx++umMGjUqJcdSkfpSdd9+++158MEHOeKII1izZg1XXHEFjz/+OAMGDOCll14q+UzssMMOjB8/niOOOILtttuOTz75hIsuuogPPviA1q1b8+STT9KrVy8aNWrEsmXLGD9+PPfff3+Kjy5w90pPdkoCopkdAZwA7OzuR5tZX6CFu09P5v31JSBu7epTQNza1ZeAuC2oKiDGXmU2sz8BdwFfAodEi/OAG+LOi4hIothLiGa2GDjM3Zea2Y/uvr2ZNQa+c/eKr8iWoRJiPFRCjI9KiPGpVyVEoDmwPHpeHNjSgY0pyIuISIlUBMQ3gSvKLDuf0OosIpIyqagytwWeA1oD7YElwHpguLuvSmYbqjLHQ1Xm+KjKHJ+qqsxpcWYEwN2/NbP9gP2BjoTq8/vuXhR3XkREEqW0H6KZlaqyJxsUVUKMh0qI8VEJMT71qlHFzPY1s3fMLAfYFD0Kor8iIikTe5UZ+BfhGuJpQG4K9i8iUqFUNKqsA1r6FuxYVeZ4qMocH1WZ41OvqszAU8CRKdiviEiVYqkym9nD/NwJuwnwlJn9ByjVzcbdT44jPyIiFYnrGmLZsd4/i2m/IiJJS8U1xF0q6oBd2fKK6BpiPHQNMT66hhif+nYN8YtKlqvUKCIplYqAWC46m1kLQHeqiEhKxdYP0cyWExpWMszs6zKrdwQejysvIiIVibNj9u8IpcMXgcSZrx1Y7e4LY8yLiEg5sQVEd58JYGat3V13qIhIvRNXP8Sr3H1c9PKKylrU3P0vceRHRKQicZUQd0143iGmfYqI1IimIZVKqR9ifNQPMT71aoBYADPrBowC2rj7H81sL6CJu3+UivyIiEBqxkMcBbxFmD6g+N7l5sBtcedFRCRRKm7d+xw4wd3nJ0xDmg6sdPedktmGqszxUJU5Pqoyx6e+3bq3M1BcNfaEvwpyIpJSqQiIcyjdMRvgBOD9FORFRKREKhpVzgdeNbPTgSwzewXYCzgiBXkRESmRkm43ZpYJDAc6AV8DL7h7drLv1zXEeOgaYnx0DTE+VV1DjC0gmtkMqr5O6O5+WDLbUkCMhwJifBQQ41Nf+iE+Usny9oRqdGaMeRERKSfOwR0eSHxtZjsCfwbOAKYAY+PKi4hIRVLRMbuFmV1PmGelDbCvu5/p7t/EnRcRkUSxBUQzyzCzPwNLgO7AAHcf7e6L48qDiEhV4ryGuJQQgG8GZgNtzKxNYgJ3nx5jfkRESokzIOYRWpnPqWS9A7vFlx0RkdLibFTpHNe+REQ2Rypu3RMRqZcUEEVEIgqIIiIRBUQRkUiDnFOloTKzM9393lTnY2un8xyfre1cq4QYrzNTnYFthM5zfLaqc62AKCISUUAUEYkoIMZrq7nWUs/pPMdnqzrXalQREYmohCgiElFAlHrBgklm9qOZvR8tO8fMVptZdjSgcF3s924zu7outr01qe48mdm1ZlbZqPgNhgLiZjCzpWZ2eJllp5jZf5J4b1LptkbRsX9sZrlmtsrM7jKzVtHqAYSZF3d19/3NLB24DTjS3Zu5+5q6yJO7n+3u19fFtuub6HP7nZllJSz7g5m9Ud17E8+TmQ00s61yQGcFRImFmV0M3ARcCrQEDiTMujjNzLaLni9195zoLW2ApsCnKcju1qwxcEGqM1EVM0vF9MiAAmKdMLMrzGyxma03s8/MbGS0vDtwN3BQVA38KVrexMxuNbOvoyri3WaWkcpjqE1m1gK4DviTu7/s7pvcfSnwP0BnYDRwPz+fl8eBhdHbfzKz6dF2upnZNDP7wcwWmtn/JOxjsplNNLMXovP+npntHq0zM/tHVDpaF5VS90l43w3R88/NbHjCNtPM7Hsz2zd6faCZzTKzn8xsvpkNrMvzVkduAS5JKJmXSOL83hCVLl8C2kX/q2wzaxcl287MHorO/6dm1jfh/e3M7P+i8/mVmZ2fsO5aM3vSzB4xs3XAKXV18NVRQKwbi4GDCSWh64BHzKytu38OnA28E1UDiz+UNwJdgd7AHoSZCP8Sf7brTD9CaW9q4sJoLu4XgcMpfV5OBHpEyVq5++DoizgNeAzYGTgBuNPM9k7Y5AmE8709Yc6ecdHyI4FDCOe4JSEQV1QFfxw4MeH1EOC/7v6hmbUHXgBuAHYALgH+z8x2quG5SLXZwBuE/JdI8vwSleCHASuj/1Uzd18ZrT4GeAJoBTwL3BFtuxHwHDCf8Nk+DLjQzIYkbPpY4MnovY/W1sHWlALi5ns6Kin8FJX07ixe4e7/z91XunuRu08BvgT2r2gjZmaE25/GuPsP7r4e+CvhA7m1aE0ILAUVrPs2Wl+d4YQq9SR3L3D3ucD/AaMS0jzl7u9H+3mU8AMDsAloDnQjdDX73N2/rWAfjwHHmFnxlLi/JQRJgN8BL7r7i9H/dRohuByVRN7rm78AfyoTzJM5v9X5T3R+CoGHgV7R8v2Andx9rLtvdPclwH2U/oy/4+5PR+c2b7OPbAulrK6+FRjh7q8VvzCzU4A/RM9PBi4iVAcBmlH5l34nwpzUc+znycqNcK1na/FfoLWZpVUQFNtG66vTCTig+DJDJI3wxSu2KuF5LuG84+7TzewOYCLQycymApe4+7rEHbj7IjP7HDjazJ4jlHj6JOx/lJkdnfCWdGBGEnmvV9z9EzN7HrgC+DxanMz5rU7Z8980uh7YiVDFTtx2Y+CthNfLa7CfOqOAWMvMrBPh1+8wwq9eoZnNIwQ5CHPHJPovYb6ZHu6+Ir6cxuodIB84Dvh38UIza0aofl2ZxDaWAzPd/YjNyYC7jwfGm9nOUR4uBSrqRlJcbW4EfObuixL2/7C7n7E5+6+HrgE+BP4eva7J+a3p3RzLga/cfc9a3GadUJW59mUR/rnfA5jZqcA+CetXA7tGLau4exEhgP4j+rJiZu3LXF9p0Nx9LeHa3gQzG2pm6WbWmRCYviG5UsjzQFczGx29P93M9osaqqoUpTvAQleeHGADUFRJ8icI1xzPIVShiz1CKDkOMbPGZtbUQveTXZPIe70TBfopQHHjRk3O72pgRzNrmeTu3gfWm9nlFqYjbmxm+5jZflt+JLVLAbGWuftnhF/ddwgfnF8AbyckmU7oSrLKzIqripcTGgHejVrZXgP2ii3TMXD3mwklwVuBdcB7hJLDYe6en8T71xMC1QnASkL17CagSRK7b0H40fkRWEZoULmlkv18S/jf9SMEjOLlywkX/q8k/NgtJ5QyG/J3aCzhB7xG59fdFxBK0kuia+jtyqYpk76QcI2yN/AVoVZ0P6GBq17RvcwiIpGG/OsmIlKrFBBFRCIKiCIiEQVEEZGIAqKISEQBUUQkooAo1TKz35iZJ7w+xcyyU5SX581scir2nUpm9kZ0+6HUIQXEBioajsmjxyYzW2JhCLGs6t+9xaYAuyWb2MLApJdUnzL1zKxzdE77Vp86VscBf051JrZ2upe5YXuNMJZgOmG4sfsJdx6cUzZhdJN9oddCT/xoNJKUjUiyLTGz7aIRYn5IdV62BSohNmz57r7K3Ze7+2OEIa9GQMmgm59E1dvFhMEVssyspZnda2Gw1PVmNrNsacjMTjazZRaG+n+eMHp14vpyVWYzO8rCoKx5ZrbGzJ6L7vd9gzDayS3FJdqE9/SL9p9rZissTCnQImF9ZlQSzrYwcG4yg0AUD+Q63cxyzGxt9LxdtG6omb1lYe6WH8zslTL3634V/f0gyu8bCds91cKAvxvM7AszG2NhrL/i9V2j49lgYYDVo6K8n5KQ5hdm9lp0nn6Ijq9lwvrJ0WWByy0M0/9NtLxUldnMtjOzm8zsm+j8fWAJ979H9yKPN7OVZpZvZsvN7MZkzt+2TAFx65JHKC0W60IY028UYWy6fMIgp+0J95b2Ad4EpptZWwAzOwCYTJhvtzdhYM+xVe3UzIYSBgSdBvwSGATMJHy+jiN8qccShvoq3s8vgFej9/WK0vUGHkzY9K2EeVZ+TRg9qA9hoNeq8tKLMCTXIqA/YaqCKfxcG8oCbieMTzkQWAs8Z9FgG/w8buXQKK/HRds9gzBO5V+A7sDFhHvQz43WNwKeAgqifZ5CGFGm5F7g6HLGK0B2tJ+RhHumE48Z4FCgZ5SHwyo51ElRut8SBg/5V3QcxWMQnh9t/wRgT+B4fh6FXCrj7no0wAchaD2f8Hp/wk3zU6LX1xIGRm2TkGYw4cuYUWZb84DLouePAdPKrL8/fFRKXp8CZCe8fht4ooq8LiWMP5i47CHggTLLehNGCtqZMJZhPnBSwvpmwE/A5Cr29Shh2LVkz2MWUAgMiF53jvLQt0y6r4HRZZZdSBgiDMLo2gVA+4T1/aJtnRK9PoMQgJsnpBkYpdkj4f/6PdCkzL7eAO6Inu9OGK2nY5k0TwN3Rs/HA68TjVegR3IPXUNs2IZGVdc0QsnwGeBPCeu/cffVCa9/SRiM9nv7eTBaCMP77x49704oFSZ6Bzi9inz0IXyRa+KXwB5mdnzCsuJM7U4YYHS7aN9AmHLAzD6uZrt9CCW1ClmYZ+V64ADC4LyNokfHKt6zE9ABuMfM7kpYlZaQ526EYfUTx7T8gNLDjHUHPvIwskyxWVGavQmlWoBPvOoRgPaN9vtZmf9jE8JoShD+H9OAL8zsVcJUDS95GG5OKqGA2LC9SZh+YBPhy7ipzPqcMq8bEYYkO7iCba2rYFldakQoef6jgnUrCPOf1IXnCVX4s6L9FACfEYJvZYovLZ1NCGB1IbGxq+z/raxGUfr9CP/7RHkAHuaB6UwouR5GqFLPN7MjFBQrp4DYsOX6zyM6J+NDQgNJkYd5LSryOeEaWKKyr8uaS/jS3VfJ+o2UnxLhQ8Io4RXmP2oI2hTte0m0LItwvWxxNXkZXMk2dySU5M519xnRsn0p/T3YGP0tya+7rzazlcDu7v5QJftdQBgmv53/POlSX0pfp/8cOM3MmieUEvtFaT4neXMJJcRdio+jItE+ngSetNB3813CJGZf1GBf2xQ1qmxbXiNc73vGzIaZWRczO8jMrjOz4lLjeOBwM/uzme0ZNSaMrGa74wjzjdxgZnubWY+oBbZ4sqalwMEWRgIvnlvmJmB/C1Ou9jGzPcxsuJndAyUz8j0A3GRmR5hZD0LjQ3VzzdwC9LHQkt7LzPayMBl7R8IAsf8Fzoj2dyhhWtjEeV6+I5SyhphZm4QW4GuAy6Lj2svCiM8nm1lx38BphEaLf0X7PRC4Ldp2cenvUcKlgIei1uZDgHuAqTX5YXP3L6JtTbbQaX43M+trZpeYWXEj0EVmdqKZdTezPQiNL+uIWq2lEqm+iKnH5j0o06hSwfprCdeiyi5vDvyT8MXYSBj5+QlC6ac4zamERoQ8why8f6SKRpVo2THAHEJDyH8JrcdNo3UHEqag3FBmO32Blwlf1BzgY2BswvosQuNLNiFQXU2o8k6u5twMIFxOyCM0wrwGtI3WDQY+ifLyCaFKmU3U8BGl+UN0/IXAGwnLTySUbDcQgut/gBMS1neN9ptPCI7Do3N8fEKaXxAaO/KibUwGWlb3fyWhUSV6nR79j5dE+1gVnfNfRuvPiPK6Pjq/M4F+qf7c1veHRswWqSNRF5h5hBbrOanOj1RPAVGklpjZSEJJ90tC953bCNf6+ri+aA2CGlVEak9zwrXRDoTq8BvAGAXDhkMlRBGRiFqZRUQiCogiIhEFRBGRiAKiiEhEAVFEJKKAKCIS+f8LMwer3y+JXgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix = confusion_matrix(y_test,y_preds)\n",
    "matrix_proportions = np.zeros((3,3))\n",
    "\n",
    "for i in range(0,3):\n",
    "    matrix_proportions[i,:] = confusion_matrix[i,:]/float(confusion_matrix[i,:].sum())\n",
    "    \n",
    "names=['Hate','Offensive','Neither']\n",
    "confusion_df = pd.DataFrame(matrix_proportions, index=names,columns=names)\n",
    "plt.figure(figsize=(5,5))\n",
    "seaborn.heatmap(confusion_df,annot=True,annot_kws={\"size\": 12},cmap='gist_gray_r',cbar=False, square=True,fmt='.2f')\n",
    "plt.ylabel(r'True categories',fontsize=14)\n",
    "plt.xlabel(r'Predicted categories',fontsize=14)\n",
    "plt.tick_params(labelsize=12)\n",
    "\n",
    "#Uncomment line below if you want to save the output\n",
    "#plt.savefig('confusion.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 298
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1964,
     "status": "ok",
     "timestamp": 1588673260312,
     "user": {
      "displayName": "Samridha Man Shrestha",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjWM1giZm-4hM3_aud8FxlOUc8VWuoYWktFbs7VDA=s64",
      "userId": "14722659845832153120"
     },
     "user_tz": -240
    },
    "id": "fNd66sKyV1mG",
    "outputId": "c83d61d5-5aa3-4e29-e042-54bdf9ec52ae"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x12a61e250>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmMAAAEICAYAAAATE/N5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dfbxeVX3n/c93QPBZQFLEEAxq7AxyW8QM4u3oUKkI1BracSzcTgmWmjLC2I69q9HeLVZLB2fa0jJaLEoKdCgPBS0ZiaUpPk3vKUhA5FFLRJCkASLhwUds8Dd/7HXClcM5OSfnnGTnXOfzfr2u19l7rbXXtfY5uVZ+19pr75WqQpIkSf34F303QJIkaS4zGJMkSeqRwZgkSVKPDMYkSZJ6ZDAmSZLUI4MxSZKkHhmMaZeQ5MAk302y2wzV9/Ekv922j0yybibqbfW9LsnXZ6o+STMjyQVJfq9t77TPaZJK8tJpHP+BJJ+cwfZ8N8mL2/aW38kM1b2lb9XMMRibQ9oHdOT14yQ/GNh/+w5835OTPDHwXt9M8udJXjZSpqq+VVXPrqonJlHX30/0nlV1alV9eIbav1VHW1X/q6p+cibqluaaJPcM9D0PtGDh2TP9PpP9nE62T5mqJF9I8sMk30nyWJIbkyxPsudAW3+/qn5lknVNWK71pXfPQNuf8ruZyb5VTzIYm0PaB/TZVfVs4FvAzw2kXTxSLsnuO+Dt/6G97/OAnwF+ANyY5JCZfqOZGl2TtMP8XOsPDgMWA//f6AI7qB/qy+lV9Rxgf+A3gBOAVUkyk28yZL+zOcVgTFsu4yV5X5L7gT8f6xvR4AhRkj2T/EGSb7Vvtx9P8oyJ3quqnqiqb1TVu4AvAh9s9S1s9e/e9k9Ocnf7NvnNJG9P8q+AjwOvad+qH2llL0hybpJVSb4H/PRYQ/PtUsC32zfztw+kb/Vtc/Dck3ypJX+1vecvjr7smeRftToeSXJ7krcM5F2Q5GNJrm7ncn2Sl0z8V5GGX1WtBz4LHAJb+pjTktwF3NXS3pzk5vb5+t9JXjFyfJJXJrmpfbYuA54+kDf6c7ogyaeSbEzyUJKPbqNP2Wb/luQ3k2xI8k9Jfnk7zvd7VfUF4C3Aa4CfbfV9MMn/aNtPT/I/WhsfSXJDkv2SnAm8Dvhoa+tHt/E7G33ZdN8kq9vv6YtJXtTKbdXvtrQvJPmVCfrb3xso/84ka5NsSrIyyQsH8irJqUnuaufysWRmA9BhYTCmES8A9gFeBCybRPmzgJcBhwIvBeYDv7Od7/kpus5lK0meBZwDHNu+Tf7fwM1VdSdwKm2Urar2Gjjs/wHOBJ4DjHXJ4QXAvq2dS4Hzkkx4CaOqXt82f6q952Wj2vo04H8Cfwv8BPCfgItH1X0C8LvA3sDa1k5pzkuyADgO+MpA8vHAq4GDk7wSWAH8KvB84M+AlS1Y2gP4a+Av6PquvwL+3TjvsxvwGeBeYCFdP3DpNvqUcfu3JMcA/y/wRmAR3Uj/dqmqbwFrGKP/o+ufngcsaOd8KvCDqvot4H/RjbI9u6pOHzhmy+9snLd8O/Bhuj7wZuDiccoNtnFb/S0ASd4A/BfgbXSjfvcCl44q9mbgXwOvaOXeNNF7z0UGYxrxY+CMqnq8qn6wrYLtm80y4D9X1aaq+g7w+3RBx/b4J7pOdLz2HJLkGVW1oapun6Cuq6rq/6+qH1fVD8cp89vt/L4IXE3XMUzXEcCzgbOq6kdV9Tm6Tv/EgTKfrqovV9Vmuk7w0Bl4X2k2++s20vL3dCPkvz+Q919av/IDun7mz6rq+jaqfiHwON3n7gjgacAfV9U/V9UVwA3jvN/hwAuB32yjUz+sqjHniU2if3sb8OdVdVtVfY82uj8F4/V//0wXhL20nfONVfXYBHUN/s7GcnVVfamqHgd+i260a8EU2z3o7cCKqrqp1f3+VvfCgTJnVdUjLQD9PPZ/YzIY04iN2whiRpsHPJNuztcjrVP9m5a+PeYDm0Yntg7uF+m+lW1ol/j+5QR13TdB/sOt3hH30nXO0/VC4L6q+vGouucP7N8/sP19uuBNmsuOr6q9qupFVfWuUUHE4Gf5RcBvjPQzra9ZQPe5eyGwvqpqoPy947zfAuDe9oVoIhP1by8c1cbx3nMiY/Z/dCN91wCXtsug/7WNwG/LRP3flvyq+m5735nq/7acf6v7Iez/tpvBmEbUqP3v0XVIACR5wUDet+km4L+8dah7VdXz2oTc7fHzdMPuT21M1TVV9Ua6oe+vAZ8Yp53jtX+0vdvlzxEH0n0zhVHnSndJc7L+CViQZPCzdCCwfjvqkPSkwc/yfcCZA/3MXlX1zKq6BNgAzB81B+nAceq8DzgwY09wH913TNS/baAL7iZ6z3G1UalXMUb/10b5freqDqabovFm4KRx2jreOYy2pb3p7lzdh67vGvmCOl7/N1G9/0QXMI/U/Sy6UT37v+1kMKbxfBV4eZJDkzydgaH4Ngr0CeDsJD8BkGR+kgnnAiTZLclBSf47cCTdXKrRZfZLsqR9sB8Hvkt32RLgAeCANl9ke/1ukj2SvI6ug/urln4z8AtJntkmvZ4y6rgHgBePU+f1dN/23pvkaUmOBH6Op86bkLT9PgGcmuTV6Twryc8meQ7wD8Bm4N3ts/cLdJcjx/JluiDqrFbH05O8tuVt1adMon+7HDg5ycFJngmcMdmTaX3MvwWuam1aNUaZn07yf7V5bo/RXbYc7P/G64u25bgk/6ad44eB66rqvqraSBc4/YfWN/8yMHiD0UT97SXAO9r/E3vSXc69vqrumUIb5zSDMY2pqv4R+BDwd3R36IyeX/E+usno1yV5rJXb1oT41yT5Ll3n8gXgucC/rqpbxyj7L4D30H3r2gT8W+A/trzPAbcD9yf59nac0v3Aw63Oi4FTq+prLe9s4Ed0Hc+FPHVy6weBC9sli63mmVXVj+iCr2PpvlH/KXDSQN2Spqiq1gDvBD5K9/ldC5zc8n4E/ELb30Q3teFT49TzBN3n9KV0j/VZ18rD2H3KuP1bVX0W+ON23Nr2cyIfTfIduj7mj4ErgWNGTW8Y8QLgCrq+8k66OXV/0fL+BHhrkoeTnDOJ9x3xl3RB4ya6Ebn/MJD3TuA36S4vvhz43wN52+xvq+rvgN9u57OBLpDb3rnDArL15XZJkiTtTI6MSZIk9chgTJIkqUcGY5IkST2aMBhLt4TE55PckW6pl19r6fukW17hrvZz75aeJOekWx7hliSHDdS1tJW/K8nSgfRXJbm1HXPOqFuVJUmShtaEE/iT7A/sX1U3tduJb6RbeuFkYFNVnZVkObB3Vb0vyXF0S8IcR7c8w59U1auT7EO3/MNiumeX3Ai8qqoeTvJl4N10jwlYBZzT7lgZ17777lsLFy6c6nlLmmVuvPHGb1fV9j5YeJdk/yXNPdvqwyZc4b2qNtDdskpVfSfJnXRP111C95wo6B4H8AW624GXABe1pyJfl2SvFtAdCayuqk0ASVYDxyT5AvDcqrqupV9EF+xtMxhbuHAha9asmaj5koZEkqk+6XyXY/8lzT3b6sO2a85YuvWmXkk3grVfC9Sge4bTfm17PlsvzbCupW0rfd0Y6WO9/7Ika5Ks2bhx4/Y0XZIkaZc06WCsLaFwJfDroxctbaNgO/yBZVV1XlUtrqrF8+YNxdUKSZI0x00qGGuLlF4JXFxVI084fqBdfhyZV/ZgS1/P1ut2HdDStpV+wBjpkiRJQ28yd1MGOB+4s6r+aCBrJTByR+RSurW2RtJPandVHgE82i5nXgMcnWTvdufl0cA1Le+xJEe09zppoC5JkqShNuEEfuC1wC8Btya5uaV9ADgLuDzJKcC9wMiafavo7qRcS7eA8jsAqmpTkg8DN7RyHxqZzA+8C7gAeAbdxP1tTt6XJEkaFpO5m/LvgfGe+3XUGOULOG2culYAK8ZIXwMcMlFbJEmSho1P4JckSeqRwZikOSnJiiQPJrltjLzfSFJJ9m37272yiCRNlsGYpLnqAuCY0YlJFtDdYPStgeRjgUXttQw4t5XdBziDbrWRw4EzRpaGk6TJMhjTrLNw+dUsXH51383QLFdVXwI2jZF1NvBetn524paVRdpqISMri7yJtrJIVT0MrGaMAE8aYf+lsRiMSVKTZAmwvqq+Oipre1cWGatuVxCRNCaDMUkCkjyT7rE9v7Mj6ncFEUnjMRiTpM5LgIOArya5h241kJuSvIDtX1lEkibNYEySgKq6tap+oqoWVtVCukuOh1XV/WznyiJ9nYOk2clgTNKclOQS4B+An0yyrq0mMp5VwN10K4t8gm7VENoqIiMri9zA1iuLSNKkTGY5JEkaOlV14gT5Cwe2t3tlEUmaLEfGJEmSemQwJkmS1CODMUmSpB4ZjEmSJPXIYEySJKlHBmOSJEk9MhiTJEnqkcGYJElSjyYMxpKsSPJgktsG0i5LcnN73ZPk5pa+MMkPBvI+PnDMq5LcmmRtknOSpKXvk2R1krvaz713xIlKkiTtiiYzMnYBcMxgQlX9YlUdWlWHAlcCnxrI/sZIXlWdOpB+LvBOYFF7jdS5HLi2qhYB17Z9SZKkOWHCYKyqvgSMudZaG916G3DJtupIsj/w3Kq6ri0rchFwfMteAlzYti8cSJckSRp6050z9jrggaq6ayDtoCRfSfLFJK9rafOBdQNl1rU0gP2qakPbvh/Yb7w3S7IsyZokazZu3DjNpkuSJPVvusHYiWw9KrYBOLCqXgm8B/jLJM+dbGVt1Ky2kX9eVS2uqsXz5s2bapslSZJ2GbtP9cAkuwO/ALxqJK2qHgceb9s3JvkG8DJgPXDAwOEHtDSAB5LsX1Ub2uXMB6faJkmSpNlmOiNjPwN8raq2XH5MMi/Jbm37xXQT9e9ulyEfS3JEm2d2EnBVO2wlsLRtLx1IlyRJGnqTebTFJcA/AD+ZZF2SU1rWCTx14v7rgVvaoy6uAE6tqpHJ/+8CPgmsBb4BfLalnwW8MclddAHeWdM4H0mSpFllwsuUVXXiOOknj5F2Jd2jLsYqvwY4ZIz0h4CjJmqHJEnSMPIJ/JIkST0yGJMkSeqRwZgkSVKPDMYkzUnjrLv735J8LcktST6dZK+BvPe3tXW/nuRNA+nHtLS1SVzOTdJ2MxiTNFddwKh1d4HVwCFV9QrgH4H3AyQ5mO4O8pe3Y/40yW7tUT4fA44FDgZObGUladIMxiTNSWOtu1tVf1tVm9vudTz5sOolwKVV9XhVfZPuET2Ht9faqrq7qn4EXNrKStKkGYxJ0th+mSefhzgfuG8gb2R93fHSJWnSDMYkaZQkvwVsBi6ewTqXJVmTZM3GjRtnqlpJQ8BgTJIGJDkZeDPw9qqqlrweWDBQbGR93fHSn6KqzquqxVW1eN68eTPebkmzl8GYJDVJjgHeC7ylqr4/kLUSOCHJnkkOolt398vADcCiJAcl2YNukv/Knd1uSbPbhMshSdIwauvuHgnsm2QdcAbd3ZN7AquTAFxXVadW1e1JLgfuoLt8eVpVPdHqOR24BtgNWFFVt+/0k5E0qxmMSZqTxll39/xtlD8TOHOM9FXAqhlsmqQ5xsuUkiRJPTIYkyRJ6pHBmCRJUo8MxiRJknpkMCZJktQjgzFJkqQeTRiMJVmR5MEktw2kfTDJ+iQ3t9dxA3nvT7I2ydeTvGkg/ZiWtjbJ8oH0g5Jc39Ivaw9OlCRJmhMmMzJ2AXDMGOlnV9Wh7bUKIMnBdE+gfnk75k+T7JZkN+BjwLHAwcCJrSzAR1pdLwUeBk6ZzglJkiTNJhMGY1X1JWDTJOtbAlxaVY9X1TeBtcDh7bW2qu6uqh8BlwJL0j3i+g3AFe34C4Hjt/McJEmSZq3pzBk7Pckt7TLm3i1tPnDfQJl1LW289OcDj1TV5lHpY0qyLMmaJGs2btw4jaZLkiTtGqYajJ0LvAQ4FNgA/OGMtWgbquq8qlpcVYvnzZu3M95SkiRph5rS2pRV9cDIdpJPAJ9pu+uBBQNFD2hpjJP+ELBXkt3b6NhgeUmSpKE3pZGxJPsP7P48MHKn5UrghCR7JjkIWAR8GbgBWNTunNyDbpL/yqoq4PPAW9vxS4GrptImSZKk2WjCkbEklwBHAvsmWQecARyZ5FCggHuAXwWoqtuTXA7cAWwGTquqJ1o9pwPXALsBK6rq9vYW7wMuTfJ7wFeA82fs7CRJknZxEwZjVXXiGMnjBkxVdSZw5hjpq4BVY6TfTXe3pSRJ0pzjE/glSZJ6ZDAmSZLUI4MxSZKkHhmMSZIk9chgTJIkqUcGY5IkST0yGJM0J7V1dR9McttA2j5JVie5q/3cu6UnyTlJ1rY1eQ8bOGZpK39XkqV9nIuk2c1gTNJcdQFwzKi05cC1VbUIuLbtAxxLt6LIImAZ3fq8JNmH7kHYr6Z7XuIZIwGcJE2WwZikOamqvgRsGpW8BLiwbV8IHD+QflF1rqNbU3d/4E3A6qraVFUPA6t5aoAnSdtkMCZJT9qvqja07fuB/dr2fOC+gXLrWtp46U+RZFmSNUnWbNy4cWZbLWlWMxiTpDFUVdGtvztT9Z1XVYuravG8efNmqlpJQ8BgTJKe9EC7/Ej7+WBLXw8sGCh3QEsbL12SJs1gTJKetBIYuSNyKXDVQPpJ7a7KI4BH2+XMa4Cjk+zdJu4f3dIkadJ277sBktSHJJcARwL7JllHd1fkWcDlSU4B7gXe1oqvAo4D1gLfB94BUFWbknwYuKGV+1BVjb4pQJK2yWBM0pxUVSeOk3XUGGULOG2celYAK2awaZLmGC9TSpIk9chgTJIkqUcGY5IkST2aMBgbZ/22/5bka22Ntk8n2aulL0zygyQ3t9fHB455VZJb29pu5yRJSx9zLThJkqS5YDIjYxfw1OU9VgOHVNUrgH8E3j+Q942qOrS9Th1IPxd4J0+u7zZS53hrwUmSJA29CYOxsdZvq6q/rarNbfc6ugcdjqs9PPG5VXVduyvpIrZe822steAkSZKG3kzMGftl4LMD+wcl+UqSLyZ5XUubT7dm24jB9dvGWwvuKVzbTZIkDZtpBWNJfgvYDFzckjYAB1bVK4H3AH+Z5LmTrW+iteBc202SJA2bKT/0NcnJwJuBo1oQRVU9Djzetm9M8g3gZXRrtQ1eyhxcv+2BJPtX1YZRa8FJkiQNvSmNjCU5Bngv8Jaq+v5A+rwku7XtF9NN1L+7XYZ8LMkR7S7Kk9h6zbex1oKTJEkaehOOjI2zftv7gT2B1e0JFde1OydfD3woyT8DPwZOHVin7V10d2Y+g26O2cg8s/HWgpMkSRp6EwZj46zfdv44Za8Erhwnbw1wyBjpDzHGWnCSJElzgU/glyRJ6pHBmCRJUo8MxiRJknpkMCZJktQjgzFJkqQeGYxJkiT1yGBMkiSpRwZjkiRJPTIYk6RRkvznJLcnuS3JJUmenuSgJNcnWZvksiR7tLJ7tv21LX9hv62XNNsYjEnSgCTzgXcDi6vqEGA34ATgI8DZVfVS4GHglHbIKcDDLf3sVk6SJs1gTJKeanfgGUl2B54JbADeAFzR8i8Ejm/bS9o+Lf+otEV7JWkyDMYkaUBVrQf+APgWXRD2KHAj8EhVbW7F1gHz2/Z84L527OZW/vmj602yLMmaJGs2bty4Y09C0qxiMCZJA5LsTTfadRDwQuBZwDHTrbeqzquqxVW1eN68edOtTtIQMRiTpK39DPDNqtpYVf8MfAp4LbBXu2wJcACwvm2vBxYAtPznAQ/t3CZLms0MxiRpa98CjkjyzDb36yjgDuDzwFtbmaXAVW17Zdun5X+uqmontlfSLGcwJkkDqup6uon4NwG30vWT5wHvA96TZC3dnLDz2yHnA89v6e8Blu/0Rkua1XafuIgkzS1VdQZwxqjku4HDxyj7Q+Df74x2SRpOjoxJkiT1aFLBWJIVSR5McttA2j5JVie5q/3cu6UnyTntadS3JDls4JilrfxdSZYOpL8qya3tmHN8Ro8kSZorJjsydgFPvbV7OXBtVS0CruXJeRLHAovaaxlwLnTBG92w/6vphvrPGAngWpl3Dhw37dvIJUmSZoNJBWNV9SVg06jkwadOj34a9UXVuY7udvD9gTcBq6tqU1U9DKwGjml5z62q69odSBcN1CVJkjTUpjNnbL+q2tC27wf2a9tbnkbdjDypelvp68ZIfwqfYC1JkobNjEzgbyNaO/y5Oj7BWpIkDZvpBGMPtEuMtJ8PtvQtT6NuRp5Uva30A8ZIlyRJGnrTCcYGnzo9+mnUJ7W7Ko8AHm2XM68Bjk6yd5u4fzRwTct7LMkR7S7KkwbqkiRJGmqTeuhrkkuAI4F9k6yjuyvyLODyJKcA9wJva8VXAccBa4HvA+8AqKpNST4M3NDKfaiqRm4KeBfdHZvPAD7bXpIkSUNvUsFYVZ04TtZRY5Qt4LRx6lkBrBgjfQ1wyGTaIkmSNEx8Ar8kSVKPDMYkSZJ6ZDAmSZLUI4MxSZKkHhmMSZIk9chgTJIkqUcGY5IkST0yGJMkSeqRwZgkSdIkLVx+NQuXXz2jdRqMSZIk9chgTJIkqUcGY5I0SpK9klyR5GtJ7kzymiT7JFmd5K72c+9WNknOSbI2yS1JDuu7/ZJmF4MxSXqqPwH+pqr+JfBTwJ3AcuDaqloEXNv2AY4FFrXXMuDcnd9cSbOZwZgkDUjyPOD1wPkAVfWjqnoEWAJc2IpdCBzftpcAF1XnOmCvJPvv5GZLmsUMxiRpawcBG4E/T/KVJJ9M8ixgv6ra0MrcD+zXtucD9w0cv66lbSXJsiRrkqzZuHHjDmy+pNnGYEyStrY7cBhwblW9EvgeT16SBKCqCqjtqbSqzquqxVW1eN68eTPWWEmzn8GYJG1tHbCuqq5v+1fQBWcPjFx+bD8fbPnrgQUDxx/Q0iRpUgzGJGlAVd0P3JfkJ1vSUcAdwEpgaUtbClzVtlcCJ7W7Ko8AHh24nClJE9p9qge2juqygaQXA78D7AW8k27OBcAHqmpVO+b9wCnAE8C7q+qaln4M3d1LuwGfrKqzptouSZoB/wm4OMkewN3AO+i+vF6e5BTgXuBtrewq4DhgLfD9VlaSJm3KwVhVfR04FCDJbnTD8p+m64jOrqo/GCyf5GDgBODlwAuBv0vyspb9MeCNdJcHbkiysqrumGrbJGk6qupmYPEYWUeNUbaA03Z4oyQNrSkHY6McBXyjqu5NMl6ZJcClVfU48M0ka4HDW97aqrobIMmlrazBmCRJGnozNWfsBOCSgf3T25OoV4w8pZrxb/+e1G3h4K3hkiRp+Ew7GGtzKt4C/FVLOhd4Cd0lzA3AH073PUZ4a7gkSRo2M3GZ8ljgpqp6AGDkJ0CSTwCfabvbuv3b28IlSdKcNBOXKU9k4BLlqGVAfh64rW2vBE5IsmeSg+jWcfsycAOwKMlBbZTthFZWkiRp6E1rZKwtEfJG4FcHkv9rkkPpnk59z0heVd2e5HK6ifmbgdOq6olWz+nANXSPtlhRVbdPp12SJEmzxbSCsar6HvD8UWm/tI3yZwJnjpG+iu5ZPZIkSXOKT+CXJEnqkcGYJElSjwzGJEmSemQwJkmS1CODMUmSpB4ZjEmSJPXIYEySJKlHBmOSJEk9MhiTJEnqkcGYJElSjwzGJEmSemQwJkm7uIXLr2bh8qv7boakHcRgTJIkqUcGY5I0hiS7JflKks+0/YOSXJ9kbZLLkuzR0vds+2tb/sI+2y1p9jEYk6Sx/Rpw58D+R4Czq+qlwMPAKS39FODhln52KydJk2YwJkmjJDkA+Fngk20/wBuAK1qRC4Hj2/aStk/LP6qVl6RJMRiTpKf6Y+C9wI/b/vOBR6pqc9tfB8xv2/OB+wBa/qOt/FaSLEuyJsmajRs37si2S5plDMYkaUCSNwMPVtWNM1lvVZ1XVYuravG8efNmsmpJs9y0g7Ek9yS5NcnNSda0tH2SrE5yV/u5d0tPknPaRNdbkhw2UM/SVv6uJEun2y5JmqLXAm9Jcg9wKd3lyT8B9kqyeytzALC+ba8HFgC0/OcBD+3MBkua3WZqZOynq+rQqlrc9pcD11bVIuDatg9wLLCovZYB50IXvAFnAK8GDgfOGAngJGlnqqr3V9UBVbUQOAH4XFW9Hfg88NZWbClwVdte2fZp+Z+rqtqJTZY0y+2oy5SDE1pHT3S9qDrX0X3T3B94E7C6qjZV1cPAauCYHdQ2SZqK9wHvSbKWbk7Y+S39fOD5Lf09PPnlU5ImZfeJi0yogL9NUsCfVdV5wH5VtaHl3w/s17a3THRtRibBjpe+lSTL6EbUOPDAA2eg6ZI0vqr6AvCFtn033cj96DI/BP79Tm2YpKEyE8HYv6mq9Ul+Alid5GuDmVVVLVCbthbonQewePFiLwNIkqRZb9qXKatqffv5IPBpum+OD7TLj7SfD7biWya6NiOTYMdLlyRJGmrTCsaSPCvJc0a2gaOB29h6Quvoia4ntbsqjwAebZczrwGOTrJ3m7h/dEuTJEkaatO9TLkf8On2sOndgb+sqr9JcgNweZJTgHuBt7Xyq4DjgLXA94F3AFTVpiQfBm5o5T5UVZum2TZJkqRd3rSCsTah9afGSH8IOGqM9AJOG6euFcCK6bRHkiRptvEJ/JIkST0yGJMkSeqRwZgkSVKPDMYk7XQLl1/NwuVX990MSdolGIxJkiT1yGBMkiSpRwZjkiRJPTIYkyRJ6pHBmCRJUo8MxiRJknpkMCZJktQjgzFJkqQeGYxJkiT1yGBMkiSpRwZjkiRJPTIYk6QBSRYk+XySO5LcnuTXWvo+SVYnuav93LulJ8k5SdYmuSXJYf2egaTZxmBMkra2GfiNqjoYOAI4LcnBwHLg2qpaBFzb9gGOBRa11zLg3J3fZEmzmcGYJA2oqg1VdVPb/g5wJzAfWAJc2IpdCBzftpcAF1XnOmCvJPvv5GZLmsWmHIxtYyj/g0nWJ7m5vY4bOOb9bSj/60neNJB+TEtbm2T5WO8nSTtbkoXAK4Hrgf2qakPLuh/Yr23PB+4bOGxdSxtd17Ika5Ks2bhx4w5rs6TZZ/dpHDsylH9TkucANyZZ3fLOrqo/GCzchvlPAF4OvBD4uyQva4txZ+oAAAa4SURBVNkfA95I14ndkGRlVd0xjbZJ0rQkeTZwJfDrVfVYki15VVVJanvqq6rzgPMAFi9evF3HShpuUw7G2jfEDW37O0lGhvLHswS4tKoeB76ZZC1weMtbW1V3AyS5tJU1GJPUiyRPowvELq6qT7XkB5LsX1Ub2mXIB1v6emDBwOEHtDRJmpQZmTM2aigf4PR2V9GKkTuOGH8of1JD/O19HOaXtEOlGwI7H7izqv5oIGslsLRtLwWuGkg/qd1VeQTw6MDlTEma0LSDsdFD+XR3Er0EOJRu5OwPp/seI6rqvKpaXFWL582bN1PVStKg1wK/BLxh1NzXs4A3JrkL+Jm2D7AKuBtYC3wCeFcPbZY0i01nztiYQ/lV9cBA/ieAz7TdbQ3lO8QvaZdQVX8PZJzso8YoX8BpO7RRkobadO6mHHMof9Qt3T8P3Na2VwInJNkzyUF0z+T5MnADsCjJQUn2oJvkv3Kq7ZIkSZpNpjMyNjKUf2uSm1vaB4ATkxwKFHAP8KsAVXV7ksvpJuZvBk6rqicAkpwOXAPsBqyoqtun0S5JkqRZYzp3U443lL9qG8ecCZw5RvqqbR0nSZI0rHwCvyRJUo8MxiRJknpkMCZJktQjgzFJkqQeGYxJkiT1aFoPfZ0tFi6/esv2PWf9bI8tkSRJ2pojY5IkST0yGJMkSeqRwZgkSVKPDMYkSZJ6ZDAmSZLUI4MxSZKkHhmMSZIk9chgTJIkqUcGY5IkST0yGJMkSeqRwZgkSVKPDMYkSZJ6tMsEY0mOSfL1JGuTLO+7PZK0PezDJE3VLhGMJdkN+BhwLHAwcGKSg/ttlSRNjn2YpOnYJYIx4HBgbVXdXVU/Ai4FlvTcJkmaLPswSVO2e98NaOYD9w3srwNePbpQkmXAsrb73SRfn2T9+wLfBshHptHKXd+W8xxy+wLf9m85++Uj23WeL9qRbZmmCfuwafRf4L/5YTIX/pYwB/6e29l/wTb6sF0lGJuUqjoPOG97j0uypqoW74Am7VI8z+ExF84R5s55wtT7L5gbv6e5cI7geQ6TmTzHXeUy5XpgwcD+AS1NkmYD+zBJU7arBGM3AIuSHJRkD+AEYGXPbZKkybIPkzRlu8RlyqranOR04BpgN2BFVd0+g28xpUsDs5DnOTzmwjnCkJynfdiMmAvnCJ7nMJmxc0xVzVRdkiRJ2k67ymVKSZKkOclgTJIkqUdDFYxNtBxJkj2TXNbyr0+ycOe3cvomcZ4nJ9mY5Ob2+pU+2jkdSVYkeTDJbePkJ8k57XdwS5LDdnYbp2sS53hkkkcH/o6/s7PbOBOSLEjy+SR3JLk9ya+NUWbW/z2ny/5rS7791ywxF/qwndZ/VdVQvOgmzX4DeDGwB/BV4OBRZd4FfLxtnwBc1ne7d9B5ngx8tO+2TvM8Xw8cBtw2Tv5xwGeBAEcA1/fd5h1wjkcCn+m7nTNwnvsDh7Xt5wD/OMa/2Vn/95zm78j+68ky9l+z5DUX+rCd1X8N08jYZJYjWQJc2LavAI5Kkp3YxpkwJ5ZdqaovAZu2UWQJcFF1rgP2SrL/zmndzJjEOQ6FqtpQVTe17e8Ad9I9sX7QrP97TpP91xCZC/0XzI0+bGf1X8MUjI21HMnoX9iWMlW1GXgUeP5Oad3Mmcx5Avy7Nlx6RZIFY+TPdpP9Pcx2r0ny1SSfTfLyvhszXe3S2iuB60dlzZW/53jsv7Zm/zU8hqYP25H91zAFY3rS/wQWVtUrgNU8+W1as8tNwIuq6qeA/w78dc/tmZYkzwauBH69qh7ruz3aZdl/DY+h6cN2dP81TMHYZJYj2VImye7A84CHdkrrZs6E51lVD1XV4233k8CrdlLbdqahX36mqh6rqu+27VXA05Ls23OzpiTJ0+g6sour6lNjFBn6v+cE7L8a+6/hMSx92M7ov4YpGJvMciQrgaVt+63A56rNvptFJjzPUdeq30J3jXvYrAROanexHAE8WlUb+m7UTErygpE5QUkOp/u8zrb/fGnncD5wZ1X90TjFhv7vOQH7r8b+a3gMQx+2s/qvXWI5pJlQ4yxHkuRDwJqqWkn3C/2LJGvpJh2e0F+Lp2aS5/nuJG8BNtOd58m9NXiKklxCdyfOvknWAWcATwOoqo8Dq+juYFkLfB94Rz8tnbpJnONbgf+YZDPwA+CEWfifL8BrgV8Cbk1yc0v7AHAgDM/fczrsv+y/+mnp9MyRPmyn9F8uhyRJktSjYbpMKUmSNOsYjEmSJPXIYEySJKlHBmOSJEk9MhiTJEnqkcGYJElSjwzGJEmSevR/ALoYlJbc8jFrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axes = plt.subplots(1, 2)\n",
    "fig.set_figwidth(10)\n",
    "\n",
    "axes[0].set_title('True Distribution')\n",
    "axes[1].set_title('Predicted Distribution')\n",
    "\n",
    "y.hist(bins=100, ax=axes[0], grid=False)\n",
    "pd.Series(y_preds).hist(bins=100, ax=axes[1], grid=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "keeJepQmSZDE"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "Logistic_regression_classifier.ipynb",
   "provenance": []
  },
  "hide_input": false,
  "kernelspec": {
   "display_name": "covid_env",
   "language": "python",
   "name": "covid"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0b2aa36b361548ce8c82a264d5e45e05": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_38ecbe6478df445887945cc61f77e527",
       "IPY_MODEL_5e6ae3dd31d34fe095867eb463744b25"
      ],
      "layout": "IPY_MODEL_19306dd9359a4d399161775bdf5e8549"
     }
    },
    "19306dd9359a4d399161775bdf5e8549": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "24c12debf83a421f9ed1ab0baebc65b0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "25991d8eb42349c398dbb6391ece0e5d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7c7575a3bbac408aafca0461affc5467",
      "placeholder": "​",
      "style": "IPY_MODEL_d022d2a3196b406291a9c366d511761a",
      "value": " 24783/24783 [00:22&lt;00:00, 1124.85it/s]"
     }
    },
    "38ecbe6478df445887945cc61f77e527": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3fe4d18ad5fe437f968dd4c4d3943b82",
      "max": 1005,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_dc80deec329c464687c79fb7a84e6401",
      "value": 1005
     }
    },
    "3fe4d18ad5fe437f968dd4c4d3943b82": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "44777498fa564a2695f6334f000bf9bf": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "48645f6e13b641b7a57276eabeb12eb4": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_859bab5cb1784f4095ad1c8b532ce68c",
       "IPY_MODEL_25991d8eb42349c398dbb6391ece0e5d"
      ],
      "layout": "IPY_MODEL_e24df47fb21d443f9bc34844ac5fa682"
     }
    },
    "5e6ae3dd31d34fe095867eb463744b25": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_24c12debf83a421f9ed1ab0baebc65b0",
      "placeholder": "​",
      "style": "IPY_MODEL_f57cb9c8ad9241d19747389f0c031c4d",
      "value": " 1005/1005 [00:01&lt;00:00, 774.78it/s]"
     }
    },
    "7c7575a3bbac408aafca0461affc5467": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "859bab5cb1784f4095ad1c8b532ce68c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_44777498fa564a2695f6334f000bf9bf",
      "max": 24783,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_f14e68c945e34132a3bddfa0452d7a16",
      "value": 24783
     }
    },
    "d022d2a3196b406291a9c366d511761a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "dc80deec329c464687c79fb7a84e6401": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "e24df47fb21d443f9bc34844ac5fa682": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f14e68c945e34132a3bddfa0452d7a16": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "f57cb9c8ad9241d19747389f0c031c4d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}