Models_Definition_&_Training_Sentiment_Analysis_Scotref2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Models_Definition_&_Training - Sentiment_Analysis_Scotref2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gracecarrillo/Political-Data-Science/blob/master/Models_Definition_%26_Training_Sentiment_Analysis_Scotref2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ma5cRQJafwRY",
        "colab_type": "text"
      },
      "source": [
        "# Scottish independence: Twitter data Sentiment Analysis\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mFiPGpFfkIUU",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "## 5. Supervised Machine Learning Models\n",
        "  - Naive Bayes\n",
        "  - Support Vector Machine\n",
        "\n",
        "## 6. Deep Learning Model\n",
        "  - LSTM Recurrent Neural Network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oP1c5gDe2M6-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Must be upgraded\n",
        "!pip install tqdm==4.36.1 --upgrade"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v0Xc0nZtWke9",
        "colab_type": "code",
        "outputId": "fadf0d0e-543a-4c49-a353-9bc0584a9602",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "!pip install --upgrade gensim "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Successfully installed gensim-3.8.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vR4yTNihbqdc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install vaderSentiment"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rk4fAdKjRIdC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cpXSoADwxTTs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# general\n",
        "import os\n",
        "import time\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import csv\n",
        "import string\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import random\n",
        "import itertools\n",
        "import collections\n",
        "from collections import Counter\n",
        "\n",
        "# tweets\n",
        "import tweepy as tw\n",
        "import re\n",
        "from collections import Counter\n",
        "from string import punctuation\n",
        "from tweepy import OAuthHandler\n",
        "import json\n",
        "\n",
        "# text manipulation \n",
        "import nltk \n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import TweetTokenizer\n",
        "from nltk.stem.wordnet import WordNetLemmatizer\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "from nltk.stem.porter import * \n",
        "\n",
        "# plots\n",
        "from wordcloud import WordCloud\n",
        "import plotly\n",
        "import chart_studio.plotly as py\n",
        "import plotly.graph_objs as go \n",
        "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n",
        "import cufflinks as cf\n",
        "cf.go_offline()\n",
        "\n",
        "# Feature Engineering\n",
        "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
        "\n",
        "# Machine Learning\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.pipeline import FeatureUnion\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
        "from sklearn.model_selection import KFold, cross_val_score\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.svm import LinearSVC\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.externals import joblib\n",
        "from sklearn.base import BaseEstimator, TransformerMixin\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer, HashingVectorizer, CountVectorizer\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "# Deep Learning\n",
        "import torch\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils.np_utils import to_categorical\n",
        "from keras.models import Sequential\n",
        "from keras import regularizers\n",
        "from keras.layers import Dense, Embedding, LSTM\n",
        "\n",
        "# For geoplots\n",
        "from IPython.display import IFrame\n",
        "import folium\n",
        "from folium import plugins\n",
        "from folium.plugins import MarkerCluster, FastMarkerCluster, HeatMapWithTime\n",
        "import networkx\n",
        "\n",
        "# hide warnings\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "\n",
        "# set plot preferences\n",
        "plt.style.use(style='ggplot')\n",
        "plt.rcParams['figure.figsize'] = (10, 6)\n",
        "pd.set_option(\"display.max_colwidth\", 200) \n",
        "\n",
        "print('Libraries imported')\n",
        "%matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fLAtJ8IhpOn3",
        "colab_type": "text"
      },
      "source": [
        "## 5. Modeling \n",
        "\n",
        "We are now done with all the pre-modeling stages required to get the data in the proper form and shape. We will be building models on the datasets with the new feature sets.\n",
        "\n",
        "We will use the following algorithms to build models:\n",
        "\n",
        "- Naive Bayes Classifier\n",
        "- Support Vector Machine\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qqysEcEkV-18",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Load back in\n",
        "train_prep = pd.read_csv('/content/drive/My Drive/Twitter_Project/feat_eng_train_data.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "90F3tCGhboeD",
        "colab_type": "text"
      },
      "source": [
        "### 5.1 Preparing data for modeling\n",
        "\n",
        "We'll first double check for missings, extract features from the train dataset and finally, split the data into train and test sets for modeling."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WF79pshTOHGY",
        "colab_type": "code",
        "outputId": "dea0defa-2a92-407c-aab2-f67e3a334c71",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Double hecking for missing values\n",
        "train_prep.isnull().values.any()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ANm6X0eNOtpT",
        "colab_type": "code",
        "outputId": "62662660-1453-4355-d429-fb392d45d737",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "train_prep.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(47824, 14)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o4koKF4WZLuO",
        "colab_type": "code",
        "outputId": "13fbedb7-3cd3-44e8-8b0e-0f89c099281b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "train_prep.head(3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>word count</th>\n",
              "      <th>tidy_tweet</th>\n",
              "      <th>tokens</th>\n",
              "      <th>neg_scores</th>\n",
              "      <th>neu_scores</th>\n",
              "      <th>pos_scores</th>\n",
              "      <th>compound_scores</th>\n",
              "      <th>NOUN</th>\n",
              "      <th>PRON</th>\n",
              "      <th>VERB</th>\n",
              "      <th>ADJ</th>\n",
              "      <th>ADV</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4.0</td>\n",
              "      <td>is it LOVE or BREAD???</td>\n",
              "      <td>5.0</td>\n",
              "      <td>love bread</td>\n",
              "      <td>['love', 'bread']</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.192</td>\n",
              "      <td>0.808</td>\n",
              "      <td>0.6369</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>now doing the weights again...urgh  my poor laptop is burning up in this summer evening's heat</td>\n",
              "      <td>16.0</td>\n",
              "      <td>weight urgh poor laptop burn summer even heat</td>\n",
              "      <td>['weights', 'urgh', 'poor', 'laptop', 'burning', 'summer', 'evening', 'heat']</td>\n",
              "      <td>0.307</td>\n",
              "      <td>0.693</td>\n",
              "      <td>0.000</td>\n",
              "      <td>-0.4767</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4.0</td>\n",
              "      <td>just done my weekly weigh in. on target  woohoo</td>\n",
              "      <td>9.0</td>\n",
              "      <td>done weekli weigh target woohoo</td>\n",
              "      <td>['done', 'weekly', 'weigh', 'target', 'woohoo']</td>\n",
              "      <td>0.000</td>\n",
              "      <td>0.548</td>\n",
              "      <td>0.452</td>\n",
              "      <td>0.5106</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   label  ...  ADV\n",
              "0    4.0  ...  0.0\n",
              "1    0.0  ...  1.0\n",
              "2    4.0  ...  0.0\n",
              "\n",
              "[3 rows x 14 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qQa10DGUZdUH",
        "colab_type": "code",
        "outputId": "fcbf8a2c-55b7-4c57-c0b8-c46ec9066fe2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Extracting Features\n",
        "features = ['tidy_tweet', 'neu_scores', 'neg_scores', 'compound_scores', 'pos_scores']\n",
        "label = ['label']\n",
        "\n",
        "# Saving features and label data in X and y for train-test split\n",
        "X = train_prep[[col for col in train_prep.columns if col in features]]\n",
        "y = train_prep[label]\n",
        "\n",
        "X.shape, y.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((47824, 5), (47824, 1))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZlOgsaulbXM2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# splitting data into training and validation set \n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=0)\n",
        "\n",
        "print(\"Shapes are {} and {}\".format(X_train.shape, X_test.shape))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V4hZEC9bcxbq",
        "colab_type": "text"
      },
      "source": [
        "### 5.2 Baseline\n",
        "\n",
        "We need a basis for comparison of results. When we start collecting results from our models, the baseline result will tell us if our models are adding value. \n",
        "\n",
        "For classification problems, the one rule is to predict the class value that is most common in the training dataset (Zero Rule Algorithm)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tcPRlfUAeD_a",
        "colab_type": "code",
        "outputId": "f48d96d6-58a6-4c91-be13-c44b3c971416",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "# Where 4 = positive and 0 = negative\n",
        "\n",
        "pd.Series(y_train.values.ravel()).value_counts(normalize=True)\n",
        "pd.Series(y_train.values.ravel()).value_counts(normalize=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0    0.504541\n",
              "4.0    0.495459\n",
              "dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aB_5h1l-iIke",
        "colab_type": "text"
      },
      "source": [
        "Our training dataset has 50.45% of instances of class `0.0` or negative and 49.54% of instances of class `4.0` or positive. Which means, it will predict *positive* and achieve a baseline accuracy of 50.45%"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z6MqvPfSihFP",
        "colab_type": "text"
      },
      "source": [
        "### 5.3 Supervised Machine Learning Models\n",
        "\n",
        "For this classification project, we'll be using two algorightms: **Naive Bayes Classifier** and **Support Vector Machine Classifier**. They are arguably two of the most used techniques for any classification task.\n",
        "\n",
        "For the source code of the helper functions and pipelines, check [this notebook](https://github.com/vtoliveira/advanced-data-science-capstone/blob/master/Model%20Definition%20and%20Training%20(Classical%20Algorithms)%20-%20Twitter%20US%20Airline%20Sentiment.ipynb)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QQeEFOK3fb24",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Helper functions\n",
        "\n",
        "class TextSelector(BaseEstimator, TransformerMixin):\n",
        "    \"\"\"\n",
        "    Transformer to select a single column from the data frame to perform additional transformations on\n",
        "    Use on text columns in the data\n",
        "    \"\"\"\n",
        "    def __init__(self, key):\n",
        "        self.key = key\n",
        "\n",
        "    def fit(self, X, y=None):\n",
        "        return self\n",
        "\n",
        "    def transform(self, X):\n",
        "        return X[self.key]\n",
        "    \n",
        "class NumberSelector(BaseEstimator, TransformerMixin):\n",
        "    \"\"\"\n",
        "    Transformer to select a single column from the data frame to perform additional transformations on\n",
        "    Use on numeric columns in the data\n",
        "    \"\"\"\n",
        "    def __init__(self, key):\n",
        "        self.key = key\n",
        "\n",
        "    def fit(self, X, y=None):\n",
        "        return self\n",
        "\n",
        "    def transform(self, X):\n",
        "        return X[[self.key]]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cjPoNZpMRfK6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Pipeline to convert tweets to a matrix of TF-IDF features.\n",
        "tfidf = Pipeline([\n",
        "                ('selector', TextSelector(key='tidy_tweet')),\n",
        "                ('tfidf', TfidfVectorizer())\n",
        "            ])\n",
        "\n",
        "# Pipeline to convert tweets to a matrix of token counts\n",
        "countvect = Pipeline([\n",
        "                ('selector', TextSelector(key='tidy_tweet')),\n",
        "                ('countvect', CountVectorizer())\n",
        "            ])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0WNt287kjzvJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Applying tfidf anf countvec to features\n",
        "\n",
        "neu_scores =  Pipeline([\n",
        "                ('selector', NumberSelector(key='neu_scores')),\n",
        "                ('minmax', MinMaxScaler())\n",
        "            ])\n",
        "neg_scores =  Pipeline([\n",
        "                ('selector', NumberSelector(key='neg_scores')),\n",
        "                ('minmax', MinMaxScaler())\n",
        "            ])\n",
        "pos_scores =  Pipeline([\n",
        "                ('selector', NumberSelector(key='pos_scores')),\n",
        "                ('minmax', MinMaxScaler())\n",
        "            ])\n",
        "\n",
        "compound_scores =  Pipeline([\n",
        "                ('selector', NumberSelector(key='compound_scores')),\n",
        "                ('minmax', MinMaxScaler())\n",
        "            ])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dQOFf5DqXN6K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# defining different sets of text processors\n",
        "def features_union(textProcessor):\n",
        "    return FeatureUnion([('text', textProcessor),\n",
        "                      ('neu_scores', neu_scores),\n",
        "                      ('neg_scores', neg_scores),\n",
        "                      ('pos_scores', pos_scores),\n",
        "                      ('compound_scores', compound_scores)])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5WeQWhLzU_t6",
        "colab_type": "code",
        "outputId": "4d1ea99c-0c72-4f6c-a69f-04eb722268d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Normalise labels\n",
        "le = LabelEncoder().fit(y_train.values.ravel())\n",
        "\n",
        "y_train = le.transform(y_train.values.ravel())\n",
        "y_test = le.transform(y_test.values.ravel())\n",
        "\n",
        "X_train.shape, X_test.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((33476, 5), (14348, 5))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6--1jZ9golSX",
        "colab_type": "text"
      },
      "source": [
        "### 5.3.1 Naive Bayes Classifier  "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "El1ahg6DbGvm",
        "colab_type": "code",
        "outputId": "e8ee642d-543a-4bcc-cc19-bcf4222bef1c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# instantiate classifier\n",
        "clf = MultinomialNB()\n",
        "\n",
        "# combine features\n",
        "features_count = features_union(countvect)\n",
        "\n",
        "# define pipeline object \n",
        "nb_pipeline = Pipeline([('features', features_count),\n",
        "                       ('nb', clf)])\n",
        "\n",
        "# Fit classifier\n",
        "nb_pipeline.fit(X_train, y_train)\n",
        "\n",
        "# score\n",
        "nb_pipeline.score(X_test, y_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7354335098968497"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-HvjG5kdtIsC",
        "colab_type": "text"
      },
      "source": [
        "### 5.3.2 Support Vector Machine Classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9lKTJiLGtPt3",
        "colab_type": "code",
        "outputId": "5e2162bd-140b-4a0f-f3fe-47562c4b9c30",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# instantiate classifier\n",
        "svm = LinearSVC()\n",
        "\n",
        "#  combine features\n",
        "features_tfidf = features_union(tfidf)\n",
        "\n",
        "# define pipeline object\n",
        "svm_pipeline = Pipeline([('features', features_tfidf),\n",
        "                       ('svm', svm)])\n",
        "\n",
        "# Fit classifier\n",
        "svm_pipeline.fit(X_train, y_train)\n",
        "\n",
        "# score\n",
        "svm_pipeline.score(X_test, y_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7303456927794815"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vewdYma7yAZj",
        "colab_type": "text"
      },
      "source": [
        "### 5.3.3 Cross-Validation\n",
        "\n",
        "For each model, we will now do a k-fold cross-validation using Exhaustive Grid Search. Cross-validation using Scikit Learn's Grid Search."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MBC-31xuys2Z",
        "colab_type": "code",
        "outputId": "2f82f205-d0d3-4e04-96da-49219a61052c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Naive Bayes Classifier\n",
        "\n",
        "# combine features\n",
        "features_tfidf = features_union(tfidf)\n",
        "\n",
        "# instantiate pipeline object\n",
        "nb_pipeline = Pipeline([('feats', features_tfidf),  ('clf', MultinomialNB())])\n",
        "\n",
        "# parameter grid (3x3x2x2x3x3x2) combinations\n",
        "parameters = {\n",
        "    'feats__text__tfidf__max_df': (0.5, 0.75, 1.0),\n",
        "    'feats__text__tfidf__ngram_range': ((1, 1), (1, 2), (2, 2)), \n",
        "    'feats__text__tfidf__use_idf': (False, True),\n",
        "    'feats__text__tfidf__binary':(False, True),\n",
        "    'feats__text__tfidf__binary':('l1', 'l2', None),\n",
        "    'clf__alpha': (1.0, 5.0, 10.0),\n",
        "    'clf__fit_prior': (True, False),     \n",
        "}\n",
        "\n",
        "# instantiate GridSearchCV object with pipeline and parameters with 3-folds cross-validation\n",
        "nb_grid = GridSearchCV(nb_pipeline, parameters, cv=3) # this takes a while :/\n",
        "\n",
        "# start time \n",
        "nb_start = time.time()\n",
        "\n",
        "# Fit \n",
        "nb_grid.fit(X_train, y_train)\n",
        "\n",
        "# end time \n",
        "svm_end = time.time()\n",
        "print(f\"Time taken to run: {round((nb_end - nb_start)/60,1)} minutes\")\n",
        "\n",
        "# Check score\n",
        "nb_grid.score(X_test, y_test)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7428909952606635"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "45BG5j0q1F0K",
        "colab_type": "code",
        "outputId": "2119ddb7-63b7-48b1-8788-fe9a5e300210",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 525
        }
      },
      "source": [
        "print('Best params: {}'.format(nb_grid.best_params_))\n",
        "nb_cv_results = pd.DataFrame(nb_grid.cv_results_)\n",
        "nb_cv_results.head(3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Best params: {'clf__alpha': 1.0, 'clf__fit_prior': False, 'feats__text__tfidf__binary': 'l1', 'feats__text__tfidf__max_df': 0.5, 'feats__text__tfidf__ngram_range': (1, 2), 'feats__text__tfidf__use_idf': False}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean_fit_time</th>\n",
              "      <th>std_fit_time</th>\n",
              "      <th>mean_score_time</th>\n",
              "      <th>std_score_time</th>\n",
              "      <th>param_clf__alpha</th>\n",
              "      <th>param_clf__fit_prior</th>\n",
              "      <th>param_feats__text__tfidf__binary</th>\n",
              "      <th>param_feats__text__tfidf__max_df</th>\n",
              "      <th>param_feats__text__tfidf__ngram_range</th>\n",
              "      <th>param_feats__text__tfidf__use_idf</th>\n",
              "      <th>params</th>\n",
              "      <th>split0_test_score</th>\n",
              "      <th>split1_test_score</th>\n",
              "      <th>split2_test_score</th>\n",
              "      <th>mean_test_score</th>\n",
              "      <th>std_test_score</th>\n",
              "      <th>rank_test_score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.229720</td>\n",
              "      <td>0.006067</td>\n",
              "      <td>0.100113</td>\n",
              "      <td>0.001856</td>\n",
              "      <td>1</td>\n",
              "      <td>True</td>\n",
              "      <td>l1</td>\n",
              "      <td>0.5</td>\n",
              "      <td>(1, 1)</td>\n",
              "      <td>False</td>\n",
              "      <td>{'clf__alpha': 1.0, 'clf__fit_prior': True, 'feats__text__tfidf__binary': 'l1', 'feats__text__tfidf__max_df': 0.5, 'feats__text__tfidf__ngram_range': (1, 1), 'feats__text__tfidf__use_idf': False}</td>\n",
              "      <td>0.738059</td>\n",
              "      <td>0.740299</td>\n",
              "      <td>0.739918</td>\n",
              "      <td>0.739425</td>\n",
              "      <td>0.000979</td>\n",
              "      <td>19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.224705</td>\n",
              "      <td>0.000837</td>\n",
              "      <td>0.102581</td>\n",
              "      <td>0.000896</td>\n",
              "      <td>1</td>\n",
              "      <td>True</td>\n",
              "      <td>l1</td>\n",
              "      <td>0.5</td>\n",
              "      <td>(1, 1)</td>\n",
              "      <td>True</td>\n",
              "      <td>{'clf__alpha': 1.0, 'clf__fit_prior': True, 'feats__text__tfidf__binary': 'l1', 'feats__text__tfidf__max_df': 0.5, 'feats__text__tfidf__ngram_range': (1, 1), 'feats__text__tfidf__use_idf': True}</td>\n",
              "      <td>0.730711</td>\n",
              "      <td>0.733847</td>\n",
              "      <td>0.731672</td>\n",
              "      <td>0.732077</td>\n",
              "      <td>0.001312</td>\n",
              "      <td>175</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.629810</td>\n",
              "      <td>0.016253</td>\n",
              "      <td>0.168072</td>\n",
              "      <td>0.002855</td>\n",
              "      <td>1</td>\n",
              "      <td>True</td>\n",
              "      <td>l1</td>\n",
              "      <td>0.5</td>\n",
              "      <td>(1, 2)</td>\n",
              "      <td>False</td>\n",
              "      <td>{'clf__alpha': 1.0, 'clf__fit_prior': True, 'feats__text__tfidf__binary': 'l1', 'feats__text__tfidf__max_df': 0.5, 'feats__text__tfidf__ngram_range': (1, 2), 'feats__text__tfidf__use_idf': False}</td>\n",
              "      <td>0.741016</td>\n",
              "      <td>0.745676</td>\n",
              "      <td>0.744130</td>\n",
              "      <td>0.743607</td>\n",
              "      <td>0.001938</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   mean_fit_time  std_fit_time  ...  std_test_score  rank_test_score\n",
              "0       0.229720      0.006067  ...        0.000979               19\n",
              "1       0.224705      0.000837  ...        0.001312              175\n",
              "2       0.629810      0.016253  ...        0.001938               10\n",
              "\n",
              "[3 rows x 17 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UmfnCDgFbaaa",
        "colab_type": "code",
        "outputId": "64e9d308-2550-4681-d779-6d8790c4a95f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Support Vector Classifier\n",
        "\n",
        "# combine features\n",
        "features_count = features_union(tfidf)\n",
        "\n",
        "# instantiate pipeline\n",
        "svm_count_pipeline = Pipeline([('feats', features_count),  ('clf', LinearSVC())])\n",
        "\n",
        "# parameter grid (3x3x2x3x7x2) combinations\n",
        "parameters = {\n",
        "    'feats__text__tfidf__max_df': (0.5, 0.75, 1.0),\n",
        "    'feats__text__tfidf__ngram_range': ((1, 1), (1, 2), (2, 2)), \n",
        "    'feats__text__tfidf__use_idf': (False, True),\n",
        "    'clf__loss': ('hinge', 'squared_hinge'),\n",
        "    'clf__C': (0.1, 0.5, 0.6, 1, 4, 5, 10, 100),\n",
        "    'clf__class_weight': (None, 'balanced')                                    \n",
        "}\n",
        "\n",
        "# instantiate GridSearchCV object with pipeline and parameters with 3-folds cross-validation\n",
        "svm_grid = GridSearchCV(svm_count_pipeline, parameters, cv=3)\n",
        "\n",
        "# start time \n",
        "svm_start = time.time()\n",
        "\n",
        "# fit\n",
        "svm_grid.fit(X_train, y_train)\n",
        "\n",
        "# end time \n",
        "svm_end = time.time()\n",
        "print(f\"Time taken to run: {round((svm_end - svm_start)/60,1)} minutes\")\n",
        "\n",
        "# score\n",
        "svm_grid.score(X_test, y_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Time taken to run: 43.0 minutes\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gMOtr-ab9jby",
        "colab_type": "code",
        "outputId": "6d885f26-2184-4dd9-c721-af799ec1a4a4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# score\n",
        "svm_grid.score(X_test, y_test)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7513242263730137"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sd_XiP2O1EsX",
        "colab_type": "code",
        "outputId": "a5dd80c4-3b50-4398-d4df-8bc0549b0349",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 473
        }
      },
      "source": [
        "print('Best params: {}'.format(svm_grid.best_params_))\n",
        "svm_cv_results = pd.DataFrame(svm_grid.cv_results_)\n",
        "svm_cv_results.head(3)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Best params: {'clf__C': 0.6, 'clf__class_weight': 'balanced', 'clf__loss': 'hinge', 'feats__text__tfidf__max_df': 0.5, 'feats__text__tfidf__ngram_range': (1, 2), 'feats__text__tfidf__use_idf': True}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean_fit_time</th>\n",
              "      <th>std_fit_time</th>\n",
              "      <th>mean_score_time</th>\n",
              "      <th>std_score_time</th>\n",
              "      <th>param_clf__C</th>\n",
              "      <th>param_clf__class_weight</th>\n",
              "      <th>param_clf__loss</th>\n",
              "      <th>param_feats__text__tfidf__max_df</th>\n",
              "      <th>param_feats__text__tfidf__ngram_range</th>\n",
              "      <th>param_feats__text__tfidf__use_idf</th>\n",
              "      <th>params</th>\n",
              "      <th>split0_test_score</th>\n",
              "      <th>split1_test_score</th>\n",
              "      <th>split2_test_score</th>\n",
              "      <th>mean_test_score</th>\n",
              "      <th>std_test_score</th>\n",
              "      <th>rank_test_score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.302991</td>\n",
              "      <td>0.041557</td>\n",
              "      <td>0.105896</td>\n",
              "      <td>0.001908</td>\n",
              "      <td>0.1</td>\n",
              "      <td>None</td>\n",
              "      <td>hinge</td>\n",
              "      <td>0.5</td>\n",
              "      <td>(1, 1)</td>\n",
              "      <td>False</td>\n",
              "      <td>{'clf__C': 0.1, 'clf__class_weight': None, 'clf__loss': 'hinge', 'feats__text__tfidf__max_df': 0.5, 'feats__text__tfidf__ngram_range': (1, 1), 'feats__text__tfidf__use_idf': False}</td>\n",
              "      <td>0.723990</td>\n",
              "      <td>0.723990</td>\n",
              "      <td>0.723875</td>\n",
              "      <td>0.723951</td>\n",
              "      <td>0.000054</td>\n",
              "      <td>217</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.254545</td>\n",
              "      <td>0.005855</td>\n",
              "      <td>0.103313</td>\n",
              "      <td>0.000916</td>\n",
              "      <td>0.1</td>\n",
              "      <td>None</td>\n",
              "      <td>hinge</td>\n",
              "      <td>0.5</td>\n",
              "      <td>(1, 1)</td>\n",
              "      <td>True</td>\n",
              "      <td>{'clf__C': 0.1, 'clf__class_weight': None, 'clf__loss': 'hinge', 'feats__text__tfidf__max_df': 0.5, 'feats__text__tfidf__ngram_range': (1, 1), 'feats__text__tfidf__use_idf': True}</td>\n",
              "      <td>0.721928</td>\n",
              "      <td>0.722377</td>\n",
              "      <td>0.723875</td>\n",
              "      <td>0.722727</td>\n",
              "      <td>0.000832</td>\n",
              "      <td>238</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.669138</td>\n",
              "      <td>0.019907</td>\n",
              "      <td>0.183208</td>\n",
              "      <td>0.008977</td>\n",
              "      <td>0.1</td>\n",
              "      <td>None</td>\n",
              "      <td>hinge</td>\n",
              "      <td>0.5</td>\n",
              "      <td>(1, 2)</td>\n",
              "      <td>False</td>\n",
              "      <td>{'clf__C': 0.1, 'clf__class_weight': None, 'clf__loss': 'hinge', 'feats__text__tfidf__max_df': 0.5, 'feats__text__tfidf__ngram_range': (1, 2), 'feats__text__tfidf__use_idf': False}</td>\n",
              "      <td>0.720943</td>\n",
              "      <td>0.719957</td>\n",
              "      <td>0.722800</td>\n",
              "      <td>0.721233</td>\n",
              "      <td>0.001179</td>\n",
              "      <td>250</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   mean_fit_time  std_fit_time  ...  std_test_score  rank_test_score\n",
              "0       0.302991      0.041557  ...        0.000054              217\n",
              "1       0.254545      0.005855  ...        0.000832              238\n",
              "2       0.669138      0.019907  ...        0.001179              250\n",
              "\n",
              "[3 rows x 17 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZiExWBn5hEf-",
        "colab_type": "text"
      },
      "source": [
        "We passed the combined parameters to the GridsearchCV object for each classifier and 10 folds for the cross validation which means that for every parameter combination, the grid ran 10 different iterations with a different test set every time.\n",
        "\n",
        "After trying out the different model parameter combinations, the GridsearchCV returns the best performing model per classifier.\n",
        "\n",
        "We save the models below to the working directory for evaluation. This is also good for when we just want to retrieve the trained model in the future without retraining it and use it for classification of new text.\n",
        " \n",
        "Also a necessary step if we plan to deploy the model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "didvyNT4hV9s",
        "colab_type": "code",
        "outputId": "7a9e97dc-df43-4b0f-8326-3a58ae3b7461",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# save best model per each classifier\n",
        "joblib.dump(nb_grid, \"/content/drive/My Drive/Twitter_Project/twitter_sentiment_naivebayes.pkl\")\n",
        "print('Naive Bayes Classifier saved')\n",
        "joblib.dump(svm_grid, \"/content/drive/My Drive/Twitter_Project/twitter_sentiment_svm.pkl\")\n",
        "print('SVM Classifier saved')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Naive Bayes Classifier saved\n",
            "SVM Classifier saved\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IbGZU7zlIv1i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Get and save predictions from best models above\n",
        "\n",
        "# Naive Bayes\n",
        "y_preds_nb = nb_grid.predict(X_test)\n",
        "# Save predictions for evaluation as numpy arrays\n",
        "np.save('/content/drive/My Drive/Twitter_Project/y_predsNB.npy', y_preds_nb)\n",
        "\n",
        "# Support Vector Machine\n",
        "y_preds_svm = svm_grid.predict(X_test)\n",
        "# Save predictions for evaluation as numpy arrays\n",
        "np.save('/content/drive/My Drive/Twitter_Project/y_predsSVM.npy', y_preds_svm)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7jiYSaEbRODC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Save test data for evaluation in the next notebook\n",
        "np.save('/content/drive/My Drive/Twitter_Project/y_test.npy', y_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EBMx_f0U6Xpx",
        "colab_type": "text"
      },
      "source": [
        "## 6. Deep Learning Modeling\n",
        "\n",
        "**Long Short-Term Memory (LSTMN) Recurrent Neural Network**\n",
        "\n",
        "We will now implement a Recurrent Neural Network (RNN). Again, our goal is to identify whether a tweet is negative or positive.\n",
        "\n",
        "The network arquitecture is as follows:\n",
        "\n",
        "  - First, we'll pass in words to an embedding layer, our first hidden layer. We need an embedding layer because we have tens of thousands of words, so we'll need a more efficient representation for our input data than one-hot encoded vectors.\n",
        "  \n",
        "  - After input words are passed to an embedding layer, the new embeddings will be passed to LSTM cells, our second hidden layer. The LSTM cells will add recurrent connections to the network and give us the ability to include information about the sequence of words in tweets.\n",
        "\n",
        "  - Finally, the LSTM outputs will go to a softmax output layer. We're using a softmax activation function and not sigmoid because we are using one hot encoding. The basic difference between Sigmoid and Softmax is that while both give output in [0,1] range, softmax ensures that the sum of outputs along channels (as per specified dimension) is 1 i.e., they are probabilities. Sigmoid just makes output between 0 to 1. Hence, if you are using a one hot encoding scheme where one channel has probabilities of one class and other channel has probabilities of another then you must use a Softmax activation.\n",
        "\n",
        "Since we are using embeddings, we will have to encode the tweet's words to integers. For this purpose, we will convert the tweets into sequences of integers using `Tokenizer` from Keras. Then, the encoded tweets can be passed into the network."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4UNCMqMpLfbE",
        "colab_type": "text"
      },
      "source": [
        "### 6.1 Encoding the words"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BjvMF1ZlhgY7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%tensorflow_version 1.x\n",
        "\n",
        "# Parameter indicating the number of words\n",
        "nb_words = 10000  \n",
        "\n",
        "## create the tokenizer (tweets have been preprocessed so no need for filters)\n",
        "tk = Tokenizer(num_words=nb_words)\n",
        "\n",
        "# fit the tokenizer on tweets\n",
        "tk.fit_on_texts(train_prep.tidy_tweet)\n",
        "\n",
        "# integer encode tweets\n",
        "tweets_seq = tk.texts_to_sequences(train_prep.tidy_tweet)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hl7AGyBsUB5n",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "Let's print out the number of unique words in the vocabulary and the contents of the first, tokenized tweet."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5ozxm63UGCK",
        "colab_type": "code",
        "outputId": "5b070280-d187-44af-bba4-04bf45268327",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "# stats about vocabulary\n",
        "print('Unique words: ', len((tweets_seq)))\n",
        "print()\n",
        "\n",
        "# print first encoded tweet\n",
        "print('Encoded tweet: \\n', tweets_seq[:1])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Unique words:  47824\n",
            "\n",
            "Encoded tweet: \n",
            " [[6, 1502]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fbSh8ZQaVrwz",
        "colab_type": "text"
      },
      "source": [
        "### 6.2 Equal lenght of sequences\n",
        "\n",
        "Each batch needs to provide sequences of equal length. We achieve this with the `pad_sequences method`. By specifying a maximum length, the sequences are padded with zeros or truncated."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GEBCCGQNYIFd",
        "colab_type": "code",
        "outputId": "12f0e9f2-6733-45e0-d6ab-a2ab7943d942",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        }
      },
      "source": [
        "train_prep['word count'].describe()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    47824.000000\n",
              "mean        13.599427\n",
              "std          6.845686\n",
              "min          0.000000\n",
              "25%          8.000000\n",
              "50%         13.000000\n",
              "75%         19.000000\n",
              "max         39.000000\n",
              "Name: word count, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VDUpPqEaXkag",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Maximum length since maximum tweet word count is 39.\n",
        "max_len = 39\n",
        "\n",
        "# Convert sequences into 2-D Numpy arrays\n",
        "features = pad_sequences(tweets_seq, maxlen=max_len)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8SpgzzFDanLA",
        "colab_type": "code",
        "outputId": "193b4045-adb0-46c3-88d3-eeaa999600a6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        }
      },
      "source": [
        "# test statements \n",
        "assert len(features)==len(tweets_seq), \"The features should have as many rows as tweets.\"\n",
        "assert len(features[0])==max_len, \"Each feature row should contain the mex length of values.\"\n",
        "\n",
        "# print first 10 values of the first 30 batches \n",
        "print(features[:10,:10])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-szktvkhZeyr",
        "colab_type": "text"
      },
      "source": [
        "### 6.4 Split into Training, Validation, and Testing sets\n",
        "\n",
        "With the tweets ready, we now split the data for training and validation. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hw518nUTWQGS",
        "colab_type": "code",
        "outputId": "ae167936-fe9f-48f6-f4c5-4600f62293b0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        }
      },
      "source": [
        "# to hot-encode\n",
        "train_prep[\"label\"] = train_prep[\"label\"].astype(\"category\")\n",
        "train_prep.label.describe()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count     47824.0\n",
              "unique        2.0\n",
              "top           0.0\n",
              "freq      24203.0\n",
              "Name: label, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Suw6f9zSTuSr",
        "colab_type": "code",
        "outputId": "67429c68-cb99-459a-fed6-56fd9ccd9b42",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        }
      },
      "source": [
        "## split data into training and test data (features and labels, x and y)\n",
        "# data needs to be as array\n",
        "labels = pd.get_dummies(train_prep['label']).values\n",
        "#labels = np.asarray(train_prep.label.values)\n",
        "\n",
        "X_train, X_test, Y_train, Y_test = train_test_split(features, labels, test_size = 0.2, random_state = 42)\n",
        "\n",
        "## print out the shapes of the resultant feature data\n",
        "print(\"\\t\\t\\tFeature Shapes:\")\n",
        "print(\"Train set X: \\t\\t{}\".format(X_train.shape),\n",
        "      \"\\nTrain set Y: \\t\\t{}\".format(Y_train.shape),\n",
        "      \"\\nTest set X: \\t\\t{}\".format(X_test.shape),\n",
        "      \"\\nTest set Y: \\t\\t{}\".format(Y_test.shape))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\t\t\tFeature Shapes:\n",
            "Train set X: \t\t(38259, 39) \n",
            "Train set Y: \t\t(38259, 2) \n",
            "Test set X: \t\t(9565, 39) \n",
            "Test set Y: \t\t(9565, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rauwUTyx0gpJ",
        "colab_type": "text"
      },
      "source": [
        "### 6.5. LSTM Network with Keras\n",
        "\n",
        " In Keras we can simply stack multiple layers on top of each other. For this we need to initialise the model as `Sequential()`.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9nI9yLfypONi",
        "colab_type": "code",
        "outputId": "b19c695b-ca83-4b83-fb38-97bdeae54c87",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# First checking if GPU is available\n",
        "train_on_gpu=torch.cuda.is_available()\n",
        "\n",
        "if(train_on_gpu):\n",
        "    print('Training on GPU.')\n",
        "else:\n",
        "    print('No GPU available, training on CPU.')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on GPU.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "5a61598c-cc5d-4f1a-d203-83ff111ad30a",
        "id": "mCOqnQI4d4iX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "#--- Parameters----#\n",
        "\n",
        "# encodes input sequence dense vectors \n",
        "embed_dim = 128\n",
        "\n",
        "# transforms the vector sequence into a single vector\n",
        "lstm_out = 200\n",
        "\n",
        "# batch size of 32 is a good starting point\n",
        "batch_size = 32\n",
        "\n",
        "# epochs\n",
        "nb_epoch = 10\n",
        "\n",
        "#------# Build the LSTM model #-----------------#\n",
        "\n",
        "print('Building model...') \n",
        "\n",
        "# Initialising the RNN\n",
        "model = Sequential()\n",
        "\n",
        "#adding an input layer and the first hidden layer\n",
        "model.add(Embedding(2500, embed_dim, \n",
        "                    input_length = features.shape[1], \n",
        "                    dropout = 0.2)) \n",
        "# Adding the second hidden layer\n",
        "model.add(LSTM(lstm_out, dropout_U = 0.2, dropout_W = 0.2))\n",
        "# Adding the output layer\n",
        "model.add(Dense(2, activation='softmax'))\n",
        "\n",
        "# Compile model\n",
        "model.compile( optimizer='adam', # optimazer\n",
        "              loss = 'categorical_crossentropy', # loss function\n",
        "              metrics = ['accuracy']) # list of metrics\n",
        "\n",
        "model.name = 'LSTM model'\n",
        "print(model.summary())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Building model...\n",
            "Model: \"LSTM model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, 39, 128)           320000    \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 200)               263200    \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 2)                 402       \n",
            "=================================================================\n",
            "Total params: 583,602\n",
            "Trainable params: 583,602\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mKSTc7teTOVl",
        "colab_type": "code",
        "outputId": "8a46d56d-cd99-4292-f426-6e4b69a7daf9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 381
        }
      },
      "source": [
        "# Fit the model\n",
        "history = model.fit(X_train, Y_train, \n",
        "                    validation_split=0.33, \n",
        "                    batch_size = batch_size, \n",
        "                    nb_epoch = nb_epoch, verbose = True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 25633 samples, validate on 12626 samples\n",
            "Epoch 1/10\n",
            "25633/25633 [==============================] - 66s 3ms/step - loss: 0.5608 - acc: 0.7091 - val_loss: 0.5233 - val_acc: 0.7428\n",
            "Epoch 2/10\n",
            "25633/25633 [==============================] - 65s 3ms/step - loss: 0.4935 - acc: 0.7623 - val_loss: 0.5232 - val_acc: 0.7405\n",
            "Epoch 3/10\n",
            "25633/25633 [==============================] - 65s 3ms/step - loss: 0.4632 - acc: 0.7777 - val_loss: 0.5343 - val_acc: 0.7378\n",
            "Epoch 4/10\n",
            "25633/25633 [==============================] - 66s 3ms/step - loss: 0.4325 - acc: 0.7952 - val_loss: 0.5585 - val_acc: 0.7319\n",
            "Epoch 5/10\n",
            "25633/25633 [==============================] - 66s 3ms/step - loss: 0.4030 - acc: 0.8100 - val_loss: 0.5701 - val_acc: 0.7298\n",
            "Epoch 6/10\n",
            "25633/25633 [==============================] - 66s 3ms/step - loss: 0.3690 - acc: 0.8264 - val_loss: 0.6362 - val_acc: 0.7175\n",
            "Epoch 7/10\n",
            "25633/25633 [==============================] - 66s 3ms/step - loss: 0.3393 - acc: 0.8414 - val_loss: 0.6652 - val_acc: 0.7100\n",
            "Epoch 8/10\n",
            "25633/25633 [==============================] - 66s 3ms/step - loss: 0.3115 - acc: 0.8551 - val_loss: 0.7608 - val_acc: 0.7083\n",
            "Epoch 9/10\n",
            "25633/25633 [==============================] - 66s 3ms/step - loss: 0.2827 - acc: 0.8713 - val_loss: 0.8198 - val_acc: 0.7027\n",
            "Epoch 10/10\n",
            "25633/25633 [==============================] - 65s 3ms/step - loss: 0.2554 - acc: 0.8843 - val_loss: 0.8845 - val_acc: 0.7014\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ltJP4o1GEs6D",
        "colab_type": "text"
      },
      "source": [
        "We ran the model for a 10 epochs. We can observe that the loss for the validation data beings to increase after the epoch 1, which suggest overfitting. Let's double check that with the help of the helper functions `eval_metric` and `optimal_epoch`. We'll then try to address the issue by tweeking our model and then test if there's any improvement. For the original source code of the helper functions, check [this very complete notebook.](https://github.com/bertcarremans/TwitterUSAirlineSentiment/blob/master/source/Handling%20overfitting%20in%20deep%20learning%20models.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GTMBOFR3Cy-s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Helper functions\n",
        "\n",
        "def eval_metric(model, history, metric_name):\n",
        "    '''\n",
        "    Function to evaluate a trained model on a chosen metric. \n",
        "    Training and validation metric are plotted in a\n",
        "    line chart for each epoch.\n",
        "    \n",
        "    Parameters:\n",
        "        history : model training history\n",
        "        metric_name : loss or accuracy\n",
        "    Output:\n",
        "        line chart with epochs of x-axis and metric on\n",
        "        y-axis\n",
        "    '''\n",
        "    metric = history.history[metric_name]\n",
        "    val_metric = history.history['val_' + metric_name]\n",
        "\n",
        "    e = range(1, nb_epoch + 1)\n",
        "\n",
        "    plt.plot(e, metric, 'ro', label='Train ' + metric_name)\n",
        "    plt.plot(e, val_metric, 'r', label='Validation ' + metric_name)\n",
        "    plt.xlabel('Epoch number')\n",
        "    plt.ylabel(metric_name)\n",
        "    plt.title('Comparing training and validation ' + metric_name + ' for ' + model.name)\n",
        "    plt.legend()\n",
        "    plt.show()\n",
        "\n",
        "\n",
        "def optimal_epoch(model_hist):\n",
        "    '''\n",
        "    Function to return the epoch number where the validation loss is\n",
        "    at its minimum\n",
        "    \n",
        "    Parameters:\n",
        "        model_hist : training history of model\n",
        "    Output:\n",
        "        epoch number with minimum validation loss\n",
        "    '''\n",
        "    min_epoch = np.argmin(model_hist.history['val_loss']) + 1\n",
        "    print(\"Minimum validation loss reached in epoch {}\".format(min_epoch))\n",
        "    return min_epoch\n",
        "\n",
        "def test_model(model, X_train, y_train, X_test, y_test, epoch_stop):\n",
        "    '''\n",
        "    Function to test the model on new data after training it\n",
        "    on the full training data with the optimal number of epochs.\n",
        "    \n",
        "    Parameters:\n",
        "        model : trained model\n",
        "        X_train : training features\n",
        "        y_train : training target\n",
        "        X_test : test features\n",
        "        y_test : test target\n",
        "        epochs : optimal number of epochs\n",
        "    Output:\n",
        "        test accuracy and test loss\n",
        "    '''\n",
        "    model.fit(X_train\n",
        "              , y_train\n",
        "              , epochs=epoch_stop\n",
        "              , batch_size=nb_epoch\n",
        "              , verbose=0)\n",
        "    results = model.evaluate(X_test, y_test)\n",
        "    print()\n",
        "    print('Test accuracy: {0:.2f}%'.format(results[1]*100))\n",
        "    return results"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CX-R0EB2DAVP",
        "colab_type": "code",
        "outputId": "1365c7f1-462b-4697-eb7d-2af668b9b6bf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        }
      },
      "source": [
        "eval_metric(model, history, 'loss')\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAGHCAYAAAAeKU4NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeVhUZf8G8HsWhh0EBlDcAzdEUANx\nKdSg0NQi09fccnszNXNpMdfs51L4upaa+ipiSqalFpVZr6hlibvhhgumKQqCLMq+zMzz+2NkchQQ\nFeYwcH+ui0vnzJlzvuc8M8PNc55zjkwIIUBEREREkpFLXQARERFRbcdARkRERCQxBjIiIiIiiTGQ\nEREREUmMgYyIiIhIYgxkRERERBJjIKMab8OGDVAqlVKX8Vj+/vtvyGQy/PHHH4/0uiZNmmDevHlV\nVJXpmGo7ZDIZoqKiHmm9w4cPR0hIyBOv+9dff4VMJsP169efeFkPU1k1P4msrCy88sorcHR0hEwm\nw99//y1pPbXJ43ye7v9sUNVhICMAQHp6OqZMmYIWLVrAysoKbm5uCAoKwsaNG6HRaKQu74kMGDAA\nN27cMMm65s2bhyZNmlTa8ho2bIjk5GQEBgY+0uuOHj2KyZMnV1odtU1V7T+lUokNGzYYTevcuTOS\nk5Ph4eFR6eurjlatWoWDBw/ijz/+QHJyMho2bFipy//oo4/g5eVV5vP5+fmYNWsWmjVrBmtrazg7\nOyMgIACfffYZAH1okclk5f4A+nArk8nQt2/fB9YRHR0NmUxmtn8IkjT4biEkJibimWeegVKpxJw5\nc9CuXTtYWFggNjYWixYtgq+vL9q2bSt1mY9MCAGNRgNra2tYW1tLXY6RoqIiqFSqh86nUChQt27d\nR16+q6vr45RFd5ly/6lUqsdqY3OVkJCA1q1bo02bNk+0nIp+hu43duxY7Nu3D59++in8/PyQlZWF\nP//8E9euXQOgD+NarRaA/ruxQ4cOiI6ORocOHR5YVqNGjfDjjz8iJSUF7u7uhulr1qxB48aNTdLr\nSTUHe8gI48aNQ2FhIU6cOIHBgwfD29sbzZo1w7Bhw3D8+HE0a9YMAFBcXIypU6eifv36UKlU8Pb2\nxubNm42WJZPJsHz5cgwYMAC2trZo1KgRtm3bhjt37mDw4MGwt7fHU089he3btxteU3JYLioqCsHB\nwbC2tsZTTz2FLVu2GC17xowZaNWqFWxsbNCwYUOMGTMGd+7cMTxfcmhy3759aNeuHSwtLRETE/PA\nIcuSxwcOHED79u1hY2ODp59+GkePHjVa3549e9CmTRtYWVnB19cXv/32W7nd9xs2bMCsWbNw9epV\nw1/SH330EQD9X90zZ87EuHHj4OLigmeffRYA8Omnn6Jt27aws7ND3bp18dprryE5OfmBfVNyyLLk\n8ddff43evXvDxsYGTz311AO9LvcfmmjSpAk+/PBDTJw4Ec7OznB3d8fkyZONej/z8/MxevRoODo6\nwsnJCePGjcO0adPK7W2oyDaUHJLbvXs3goKCYGNjA29vb+zatctoOSdPnkTnzp1haWmJZs2a4euv\nvy53vVlZWbCxsXngPZiUlASlUomYmBgAwObNmxEYGAhHR0eo1Wr06tULFy9eLHfZ9++/jIwMw3va\n3d0dM2fOxP03Odm9eze6desGZ2dnODo6omvXrjhy5IjRMrVaLUaMGGHU01LaIctDhw4hKCgI1tbW\ncHJywqBBg5Cammp4vqQXKDo6Gi1btoStrS26deuGhISEcrfrfkIILFq0CE899RRUKhU8PT2xbNky\no3mio6PRrl072NjYoE6dOujQoQP+/PNPAPrvhHfeeQcNGjSApaUl6tWrh9dee63c/RoREYG9e/dC\nJpOhW7duAIDs7Gy8+eabcHV1haWlJfz9/fG///3P8LqS9/2XX36JF198Eba2tpg1a9YjbWuJ7777\nDu+//z7CwsLQtGlT+Pn5Yfjw4fjwww8B6MN43bp1UbduXUMwd3Z2Nky7Nzw3a9YMHTt2NPr8Xbt2\nDbt378aIESMeWku3bt0watQozJw5E25ubqhTpw5mzJgBnU6HOXPmwN3dHa6urpgxY4bR6x62v4CK\nfZ5ycnIwceJE1K9fHzY2NmjXrh127NhR4X1JlUxQrZaeni7kcrmYO3fuQ+d97733hLOzs/j666/F\nhQsXxPz584VMJhMxMTGGeQAId3d3sWHDBpGQkCDGjh0rrKysRI8ePURkZKRISEgQ48ePFzY2NiIt\nLU0IIcSVK1cEAFGvXj0RFRUlzp8/L2bMmCHkcrk4ceKEYdlz584V+/fvF1euXBExMTGiRYsW4vXX\nXzc8HxkZKWQymQgICBB79+4Vf/31l0hNTRWRkZFCoVA8MN+zzz4r9u/fL86dOyd69OghmjRpIoqL\ni4UQQly/fl1YW1uLUaNGibNnz4qYmBjRrl07AUBs2rSp1P2Tl5cnPvjgA9GgQQORnJwskpOTRXZ2\nthBCiMaNGwt7e3sxe/ZsceHCBXH27FkhhBDLli0Tu3fvFpcvXxaxsbGiU6dOIigoyLDMkn3z+++/\nGz1u2rSp2Lp1q0hISBDTpk0TCoVCXLhwwfC6xo0bG7Vp48aNRZ06dcQnn3wiLl68KLZu3SqUSqVY\nt26dYZ63335buLm5iejoaHH+/HkxdepU4eDgIDw9Pct9XzxsG/bt2ycACF9fX7Fr1y5x8eJFMXz4\ncGFvby8yMjIM+87Dw0P07NlTxMXFidjYWOHv7y+sra3LfW8OHDhQ9OjRw2jaggULRIMGDYRWqxVC\nCLF+/Xrx/fffi0uXLokTJ06IPn36CC8vL1FYWGh4zf3tev/+CwsLE56enmLPnj3izJkzYvDgwcLe\n3l4EBwcb5tmxY4fYunWrOH/+vDhz5owYNWqUcHJyMrzPU1NThUKhEMuWLTO8P+7dP4mJiUIIIZKT\nk4W9vb0YOHCgOHXqlPj9999FmzZtxLPPPmtY1+zZs4WNjY0IDQ0Vx44dE3FxcaJ9+/bimWeeKbet\nhg0bZlTzihUrhJWVlVizZo24ePGiWLVqlbC0tDS8L5KTk4WFhYVYsGCBuHz5soiPjxdffvmlOHXq\nlBBCiMWLF4v69euLffv2iatXr4ojR46IpUuXlrn+1NRU8a9//Us8++yzIjk5WaSnpwshhOjXr59o\n3Lix+Pnnn0V8fLyYMGGCsLCwEOfOnRNC/PO+r1+/voiKihKXL18Wly9fLnUds2fPLvc927JlS9Gr\nVy/Dustz/+fvXiX7ctOmTcLLy0vodDohhBCzZs0SoaGhD3zvlKZr167CwcFBTJkyRVy4cEFEREQI\nAKJHjx7i/fffFxcuXBAbNmwQAMRPP/1keN3D9ldFPk86nU5069ZNdO3aVfz+++/ir7/+EmvWrBEW\nFhYPfKeX9Z1HlYuBrJY7fPiwACC2b99e7ny5ublCpVKJlStXGk0PCwsT3bt3NzwGICZOnGh4nJqa\nKgCI8ePHG6ZlZGQIAOKHH34QQvzzpTdz5kyjZXfq1EkMGTKkzJp27NghVCqV4RdvZGSkACD2799v\nNF9pgQyAOH78uGHaoUOHBABx/vx5IYQQ06dPF40bNxYajcYwz65dux765TR37lzRuHHjB6Y3btxY\nPPfcc2W+rsSJEycEAHH9+nUhRNmBbPHixYbXaDQaYWdnJ1avXm20vvsDWZ8+fYzW1aNHD/Haa68J\nIYTIyckRKpXKKKAJIURgYOBDA9nDtqEkcNz7Hrt586YAIH7++WchhBBr164Vtra2hoAmhBCnT58W\nAMoNZLt27RIKhcIQboQQwsfHR0ydOrXM16SnpwsA4o8//jBMKy+QJSQkCADif//7n+H5wsJC4eHh\nYRRu7qfVakWdOnVEVFSUYZpCoRCRkZFG890fyGbOnCnq169vFBjj4uIEAPHbb78JIfShQ6FQiNTU\nVMM8W7ZsETKZTOTn55dZ0/2BrEGDBuL99983mmfSpEmiadOmQoh/2vLKlSulLm/ChAmie/fuhjBS\nEffXULJ/d+7caTRfu3btxIgRI4QQ/7zv58yZ89DlPyyQ/fHHH6JRo0ZCLpeLNm3aiDfeeEN8++23\npW5DRQJZfn6+cHZ2Fnv37hUajUbUr19fbN++vcKBzM/Pz2iat7e38PHxMZrm6+sr3n33XSFExfZX\nRT5P+/btE5aWluL27dtGyxkxYoR4+eWXDY8ZyEyHhyxrOVHBe8tfunQJRUVFCAoKMpretWtXnD17\n1mian5+f4f+urq5QKBTw9fU1THNycoJKpTI6BAMAnTp1MnrcpUsXo2Xv2LEDQUFB8PDwgJ2dHQYP\nHoyioiLcvHnT6HUBAQEP3R6ZTGZUZ8mA6pSUFABAfHw8AgICoFAoyqzvUZU2BuXXX39FaGgoGjZs\nCHt7ezzzzDMAgKtXr5a7rHvH9CkUCri5uRlqr8hrAP02l7ympH07duxoNE9Ftrmi23Dv+t3d3aFQ\nKIz2d6tWreDk5GSYx8fHB46OjuWu+/nnn4ebm5vhsOWJEydw5swZvP7664Z54uLi8Morr6Bp06aw\nt7dHo0aNSq2vLPHx8QD0g+9LqFSqB95nV65cwdChQ+Hl5QUHBwc4ODjgzp07FV5PibNnz6Jjx45G\n46P8/Pzg6Oho9Hnw8PAwGuvm4eEBIcQDn6uyZGVl4fr166V+pv/++2/k5eXB19cXoaGh8PHxwSuv\nvIJPP/0UiYmJhnlHjBiB06dPw8vLC2PGjMH27dtRVFT0SNtbsn/vryMoKOiB75bSPkOPqkuXLvjr\nr7/w+++/Y9iwYUhJSUG/fv3w0ksvVfj78F5WVlYYOnQo1q5di507d0Kj0aBPnz4Vfv2930MAULdu\nXaPvy5JpJe1akf1Vkc/T0aNHUVRUhPr168POzs7wExUV9ciHvqlyMJDVcs2aNYNcLjd8yCuDhYXF\nQ6fJZDLodLoKL/Pw4cPo378/goKC8O233+LEiRNYvXo1ABj9AlAoFLCysnro8uRyuVHYKhnPc29N\nJdMqi62trdHja9eu4cUXX0STJk2wZcsWHDt2DN9//z0APPSX2v2DmSuyPyvymkfd5kfZhtIGYD/K\ne6A0CoUCgwcPxsaNGwEAGzduREBAAFq1agUAyMvLwwsvvACZTIbIyEgcOXIER48ehUwme+Tg8DC9\ne/fGtWvXsHLlShw6dAhxcXFwc3Or9PWUKK09gSffp/dSKBTYtWsX9u7di4CAAGzfvh3NmzfHjz/+\nCEAfsq9cuYJFixZBpVJh4sSJaNu2LbKysiqthnvd/xl6XEqlEp07d8a7776L6OhobNiwAT/++CP2\n79//WMsbPXo0duzYgYULF2LEiBGlfgeWpbTvxif9vqwInU4HR0dHxMXFGf3Ex8c/ML6TTIOBrJZz\ndnZGz549sWLFCqMB8iWKi4uRm5sLLy8vWFpaPvCF9dtvv8HHx6dSajl06JDR49jYWHh7ewMA/vjj\nD6jVasybNw+BgYFo3rx5lZ7B5O3tbXS2VWn1lUalUhm9pjxHjx5Ffn4+li1bhi5duqBFixYP7eWq\nKl5eXlCpVDh48KDR9Idtc2Vtg7e3N86dO4fbt28bpp09e7bU9+T9hg0bhpMnT+LPP//EV199ZdQ7\ndu7cOdy6dQvz589Ht27d0KpVK2RmZj5ST0jJezA2NtYwraioyOgkkPT0dMTHx2Pq1KkIDQ2Ft7c3\nrKysHuitqsj7o3Xr1jh06JBRkDt58iTu3LlTaZ81AHBwcECDBg1K/Uw3bdoUNjY2APRhoEOHDpg+\nfTr279+Prl27IjIy0jC/nZ0dXnnlFXz22Wc4duwYzp07h99++63CdbRu3RoAHqhj//79lbq95SkJ\n8BXtXbyft7c3AgICcODAAfz73/+uzNIeUJH9VZHPk7+/P27fvo2CggJ4eXkZ/ZT0IpNpMZARPv/8\nc1hYWODpp5/G5s2bER8fj0uXLiEqKgr+/v5ISEiAjY0NJkyYgFmzZuGbb77BxYsX8fHHHyM6OhrT\np0+vlDoiIiKwefNmXLx4ER9++CEOHjyId955BwDQokUL3Lp1CxEREbh8+TI2btyIzz//vFLWW5px\n48YhJSUFY8eOxblz57Bv3z7DmU7l9SI1bdoUN2/exMGDB5GWloa8vLwy523WrBlkMhkWL16MK1eu\n4LvvvsOcOXMqfVsqwtbWFm+++SZmzpyJH3/8ERcvXsSMGTNw7ty5cre3srZh0KBBsLe3x5AhQ3Dy\n5EkcOnQII0eOrNDlSnx8fNCuXTuMHDkSt2/fxsCBAw3PNW7cGJaWlli+fDn++usv7NmzBxMnTnyk\nnkAvLy+89NJLeOutt7Bv3z7Ex8fj3//+N7Kzsw3zODk5wdXVFWvXrsXFixdx8OBBDBw48IH6mzZt\nin379iEpKQlpaWmlrm/8+PHIysrC8OHDcebMGfzxxx8YOnQonn32WcPZuZVl2rRpWL58OdauXYuE\nhASsWbMGq1atMnymY2NjMXfuXBw+fBjXrl3Dnj17cOrUKUNIXbhwIb788kucPXsWV65cwfr166FQ\nKNC8efMK1+Dp6Yn+/ftj3Lhx+OWXX3D+/HlMnDgRZ86cwfvvv/9Y21VUVPRAz8+pU6cA6A/Jrl69\nGseOHcPVq1exZ88ejBs3DnXq1EH37t0fa30A8MsvvyAtLQ2enp6PvYyKqMj+qsjn6bnnnkNISAj6\n9u2L7777DpcvX8bx48cN7wcyPQYyQqNGjXDixAmEhYXho48+Qvv27dG5c2esXbsW77//vuGvrvnz\n5+ONN97ApEmT4OPjg6ioKMOlKipDeHg4/vvf/8LX1xebNm1CVFQU2rdvD0B/OGjGjBmYPn062rRp\ngy1btmDhwoWVst7S1K9fH99//z1iY2PRtm1bTJw4EXPnzgWAcg+JhoWFoX///ujVqxdcXV3xn//8\np8x5fX19sXz5cqxZswbe3t5YtGjRA5ccMKUFCxagT58+GDRoEDp06IDMzEwMHz683O2trG2wsbHB\nTz/9hPT0dHTo0AGDBw/G5MmT4ebmVqHXDxs2DHFxcXjxxRfh4uJimK5WqxEVFYXdu3ejdevWeO+9\n97Bo0SLI5Y/21bd+/Xq0bdsWvXv3RteuXVG/fn288sorhuflcjm++eYb/PXXX/D19cXw4cMxadIk\n1KtXz2g5ixcvxvHjx9GkSZMyr3Xm7u6O//3vf7h+/ToCAgLQu3dv+Pj4YNu2bY9Uc0WMHTsWc+bM\nwccffwxvb28sWLAA4eHhGDVqFADA0dERBw8exMsvv4xmzZph5MiRGDx4sOGSEw4ODliyZAk6deqE\nNm3a4Ntvv8X27dvRokWLR6pj3bp1CA0NxZAhQ+Dn54cDBw7gxx9/RMuWLR9ruxITE9GuXTujn5Lx\nZz179jRcPqNFixYYMWIEmjVrhgMHDkCtVj/W+gD9e9jZ2fmxX/8oHra/KvJ5kslk+P7779G3b19M\nnjwZLVu2RK9evbBz584qD5VUOpl4nFGMRJXo77//RtOmTfH7778bBoRXRyWHa06dOvXEF7U0F889\n9xycnJyMrhtHRESVj1fqJyrDqlWr4OfnBw8PD8THx2Py5MkIDAyssWHs9OnTOHHiBDp16oSioiJs\n2rQJ+/bt4wBfIiITYCAjKsPVq1fxySefICUlBXXr1sXzzz+PBQsWSF1WlZHJZFi1ahUmTJgAnU6H\nli1b4ttvv0WPHj2kLo2IqMbjIUsiIiIiiXFQPxEREZHEGMiIiIiIJMZARkRERCQxsx/Un5SUJHUJ\nZk+tVpd5kUoyD2xD88b2M39sQ/NnijYsuW9yadhDRkRERCQxBjIiIiIiiZnskGVcXBwiIyOh0+kQ\nHByMsLAwo+dv3bqFVatWISsrC3Z2dnj77beNboFCREREVFOZJJDpdDpERERg5syZcHFxwbRp0+Dv\n748GDRoY5tm0aROCgoLQrVs3nDlzBps3b8bbb7/9yOsSQqCgoAA6ne6RbiBcm6WkpKCwsNDk6xVC\nQC6Xw8rKim1FRES1mkkC2aVLl1C3bl24u7sDADp37oyjR48aBbLr16/j9ddfBwC0bt36sW8cXVBQ\nAAsLCyiVZn++gskolUooFApJ1q3RaFBQUABra2tJ1k9ERFQdmCS1ZGRkGB1+dHFxQUJCgtE8jRs3\nxpEjR/Diiy/iyJEjyM/PR3Z2Nuzt7Y3mi4mJQUxMDAAgPDwcarXa6PmUlBRYWlpW0ZbUXFIFWKVS\nCZlM9kA70qNRKpXch2aM7Wf+2IbmT+o2rDbdSEOHDsX69evx66+/olWrVnB2doZc/uA5ByEhIQgJ\nCTE8vv8U1cLCQsl6e8yVUqmERqORbP2FhYU8XfwJ8ZR788b2M39sQ/Mn9WUvTBLInJ2dkZ6ebnic\nnp4OZ2fnB+Z57733AOgPOx4+fBi2tramKK9SZWRkYMCAAQD0JyooFArDtu7cuRMqleqhy5g8eTLe\neusteHl5VWidmzdvxvnz5zFnzpzHL5yIiIgkY5JA5unpieTkZKSmpsLZ2RmxsbGYMGGC0TwlZ1fK\n5XJ8++236N69uylKg/WOHbAPD4ciKQlaDw9kT52K/L59H3t5zs7O2L17NwBg8eLFsLW1xZgxY4zm\nEUIYBrSXZunSpY+9fiIiIjI/JglkCoUCI0eOxPz586HT6dC9e3c0bNgQW7duhaenJ/z9/REfH4/N\nmzdDJpOhVatWGDVqVJXXZb1jBxynTIE8Px8AoLxxA45TpgDAE4Wy0ly5cgUjRoyAj48Pzpw5g6++\n+gpLly7F6dOnUVBQgJdeegmTJ08GAISFhWHevHlo2bIl2rRpg6FDh2Lv3r2wtrZGZGRkuce4r127\nhnfeeQe3b9+GWq3GkiVL4OHhgejoaHz66aeQy+WoU6cOtm3bhnPnzuHdd9+FRqMxnAnbuHHjSt1u\nIiIiejiTjSFr37492rdvbzSt5NAeAHTs2BEdO3Y0VTkAAPvwcEMYKyHPz4d9eHilBzJAf7bpp59+\nCj8/PwDAtGnT4OTkBI1Gg/79+6NXr15o3ry50WuysrLQsWNHTJ8+HR999BG2bNmC8ePHl7mO6dOn\nY9CgQejbty+ioqIwe/ZsrF27FkuWLMG2bdvg6uqKO3fuAAC++OILvPnmm3j11VeRm5sLIUSlbzMR\nERE9XK2+Ur+ijPtgljX9STVu3NgQxgAgOjoaoaGh6NGjBxISEnDx4sUHXmNlZYXnnnsOAODr64vE\nxMRy1/Hnn3/i5ZdfBgD069cPR44cAQAEBARg4sSJ2Lx5M3Q6HQDA398fn332GVasWIGkpCRYWVlV\nynYSERGZDSFg+dtvQEqKpGXU6kCmLeNsh7KmPykbGxvD/y9fvox169bh66+/RkxMDLp3717qxVnv\nPQlAoVBAq9U+1roXLlyId999F4mJiejRowdu376Nfv36Yd26dVCpVBgyZAgOHTr0WMsmIiIyRxan\nTsHltdfgMmgQFCtXSlpLrQ5k2VOnQnffBUl11tbInjq1ytedk5MDOzs72NvbIyUlBb/++mulLLd9\n+/b44YcfAAA7duxAYGAgAODq1at4+umnMWXKFDg6OuLmzZu4evUqmjZtitGjRyM4OBjnzp2rlBqI\niIiqM8XVq6gzbhxce/aEMj4ed+bMgXbGDElrqjbXIZNCyTixyjzLsqLatGmDZs2aISgoCA0aNEBA\nQEClLHf+/Pl45513sGLFCsOgfgD46KOPkJiYCCEEgoKC0LJlSyxbtgzR0dGwsLCAu7s73n333Uqp\ngYiIqDqSp6fDbtky2G7aBKFUInviROSMHQthbw9rS0sgO1uy2mTCzEdyJ9033isvL8/o0CA9nNQX\nhmWbPTlelNK8sf3MH9uwepPl5cF2zRrYrV4NWX4+8gYORPY770B395aOQC25MCwRERGRyRUXw+ar\nr2C/dCkUqanI79kT2VOnQlPBC6+bEgMZERER1SxCwOqnn+AQHg7l5cso7NABGWvXotjfX+rKysRA\nRkRERDWG6vBhOMybB9WJEyhu3hzpkZEofP55QCaTurRyMZARERGR2VNeuACHjz+GVUwMtHXrInPx\nYuT36wcozSPqmEeVRERERKWQJyXBYdEiWH/zDYSdHbKmT0fuyJEQ913WqrpjICMiIiKzI7t9G3Yr\nV8Ju/XpAp0PuG28ge/x4CGdnqUt7LLX6wrBVoV+/fg9c5HXt2rWY+pCLzTZr1gwAcPPmTbzxxhtl\nLvvkyZPlLmft2rXIv+f+nEOHDjXcu/JJLF68GKtXr37i5RARET2RggLYrl4N9y5dYLdqFfJ79ULq\n778j68MPzTaMAQxklS4sLAzR0dFG06KjoxEWFlah19etWxdr16597PWvW7fOKJBt2rQJjo6Oj708\nIiKiakGrhfU338AtKAiOc+eiqF073PrlF9z+7DNoGzSQuronxkBWyXr16oU9e/agqKgIAJCYmIiU\nlBQEBgYiNzcX//rXvxAaGorg4GD88ssvD7w+MTHRcDPx/Px8jB07Fl27dsWoUaNQUFBgmG/q1Kno\n2bMnunfvjkWLFgEAIiIikJKSgv79+6Nfv34AgMDAQGRkZAAA1qxZg+eeew7PPfecIfQlJibimWee\nwfvvv4/u3btj4MCBRoGuNGfOnEHv3r0REhKCUaNG4fbt24b1d+vWDSEhIRg7diwA4ODBg3j++efx\n/PPP44UXXkBOTs5j71siIqqFhIDl3r1wDQ2F06RJ0KnVSNu6FRlRUdC0bi11dZWmRo8hc/jwQ1jE\nx1fqMou9vZE1Z06Zzzs5OaFt27bYt28fQkNDER0djT59+kAmk8HS0hIRERGwt7dHRkYG+vTpgxde\neAGyMk7F3bhxI6ytrfHbb78hPj4ePXr0MDz3wQcfwMnJCVqtFgMGDEB8fDxGjRqF//73v/jmm2/g\nfF+37alTp/D111/jxx9/hBACvXv3RqdOneDo6IjLly9jxYoVWLhwId5880389NNPePXVV8vcxkmT\nJmHu3Lno1KkTFi5ciCVLlmDOnDlYuXIlDh48CEtLS8Nh0tWrV+Pjjz9GQEAAcnNzYWlp+Si7m4iI\najGLuDg4zJsHy4MHoWnSBBmrVqGgd29AXvP6k2reFlUD9x62vPdwpRAC4eHhCAkJwYABA3Dz5k3c\nunWrzOUcPnwYfe/eV9Pb28S+WlMAACAASURBVButWrUyPPfDDz8gNDQUoaGhuHDhAhISEsqt6ciR\nI+jRowdsbGxga2uLnj174vDhwwCARo0awcfHBwDg6+uLxMTEMpeTlZWFO3fuoFOnTgCA/v37G5bT\nqlUrjB8/Htu3b4fy7mnGAQEB+L//+z9ERETgzp07hulERERlUVy5Aqc334Rrr15QXryI2/PnI3Xf\nPhS89FKNDGNADe8hK68nqyqFhobio48+wunTp5Gfnw9fX18AwI4dO5Ceno5du3bBwsICgYGBKCws\nfOTlX7t2DWvWrMHOnTtRp04dTJo0yehw5qNSqVSG/ysUisde1saNG3Ho0CHs3r0bn332Gfbs2YPx\n48cjODgYe/fuRVhYGDZv3gyvanjLCiIikp781i3YL10Kmy+/hFCpkD15MnLGjIGws5O6tCpXM2Om\nxGxtbdG5c2e88847RoP5s7OzoVarYWFhgQMHDuD69evlLicwMBDfffcdAOD8+fM4d+6cYTnW1tZw\ncHDArVu3sG/fPsNr7OzsSh2nFRgYiF9++QX5+fnIy8vDzz//jMDAwEfeNgcHBzg6Ohp6xbZv346O\nHTtCp9MhKSkJXbp0wYwZM5CdnY3c3Fz8/fffaNWqFd566y34+fnh0qVLj7xOIiKq2WQ5ObBbsgRu\nXbrAJioKeYMGIfXAAWS/916tCGNADe8hk1JYWBhGjRqFVatWGab17dsXw4YNQ3BwMHx9fR/aU/T6\n66/jnXfeQdeuXdGsWTNDT1vr1q3h4+ODoKAgeHh4ICAgwPCawYMHY/DgwXB3d8e2bdsM09u0aYP+\n/fujV69eAICBAwfCx8en3MOTZVm2bBmmTp2KgoICNGrUCEuWLIFWq8Xbb7+N7OxsCCEwcuRIODo6\nYuHChYiNjYVcLkfz5s3RvXv3R14fERHVUMXFsPnyS/3Nv9PSkN+rF7I++ABaT0+pKzM5mRBCSF3E\nk0hKSjJ6nJeXBxsbG4mqMU9KpRIajUay9bPNnpxarUZaWprUZdBjYvuZP7bhIxICVj/+qL/5999/\no7BTJ2RNn47i9u0lK8kUbejh4VHmc+whIyIiIpNRxcbCYf58qOLiUNyyJdI3bkThc89V+5t/VzUG\nMiIiIqpyyvh4OHzyCaz27oW2Xj1kLlmiv/m3QiF1adUCAxkRERFVGcWNG7BfuBDW27ZBODjgzsyZ\nyB0+HDCzm39XtRoXyMx8SFytxDYjIqp5ZJmZsF+xAraRkQCA3DFj9Df/rlNH4sqqpxoXyORyOTQa\nDS9AaiY0Gg3kNfQif0REtVJ+PuwiI2G3YgVkWVnI798fWe+9B139+lJXVq3VuNRiZWWFgoICFBYW\nlnlLIjJmaWn5WBeofVJCCMjlclhZWZl83UREVMm0Wlhv2waHhQuhSE5GQXAwsqZNg+aeu8xQ2Wpc\nIJPJZLDmcelHwtO1iYjosQkBy5gYOHzyCSwuXEBRu3bIXL4cRXdvsUcVU+MCGREREZmGxfHjcPj4\nY1geOgRN06bIWLMGBb161fpLWDwOBjIiIiJ6JIpLl+CwYAGsf/oJWldX3P74Y+QNGgRYWEhdmtli\nICMiIqIKkaek6G/+vXkzhJUVst57D7mjR0PY2kpdmtljICMiIqJyybKzYbd6NWzXrIGsuBi5r7+O\nnEmToFOrpS6txmAgIyIiolLJ8vJgs3Ej7D7/HIr0dOS/9BKypkyBtmlTqUurcRjIiIiIyIgsNxe2\nX3wB29WroUhPR+GzzyJj6lQUt20rdWk1FgMZERERAdAfmrTdsAG2a9ZAkZmJgm7dkDFpEooDAqQu\nrcZjICMiIqrlZFlZsI2IgN26dZDfvo2C4GB9EGvfXurSag0GMiIiolpKdvs27CIiYLtuHeRZWch/\n4QXkTJqEYj8/qUurdRjIiIiIahlZRgbs1q2D7fr1kGdnI79nT2RPmgSNj4/UpdVaDGRERES1hDwj\nA7Zr1sA2MhLy3Fzk9+qlD2Le3lKXVuuZLJDFxcUhMjISOp0OwcHBCAsLM3o+LS0NK1euRG5uLnQ6\nHQYNGoT2PHZNRET0xORpabBbvRo2X3wBWX4+Cvr0QfbEidC0bCl1aXSXSQKZTqdDREQEZs6cCRcX\nF0ybNg3+/v5o0KCBYZ7t27ejU6dOeOGFF3D9+nV88sknDGRERERPQJ6aCrtVq2CzcSNkRUXIDwtD\nzoQJ0DRrJnVpdB+TBLJLly6hbt26cHd3BwB07twZR48eNQpkMpkMeXl5AIC8vDw4OTmZojQiIqIa\nR37zJuw+/xy2X34JFBUhv29fZL/9NrReXlKXRmUwSSDLyMiAi4uL4bGLiwsSEhKM5unfvz/mzZuH\nn3/+GYWFhZg1a1apy4qJiUFMTAwAIDw8HGretuGJKZVK7kczxzY0b2w/81dt2vD6dSgWLYJ8/XpA\no4FuyBBop0yB0ssL7OYon9RtWG0G9R84cADdunVDnz59cPHiRSxfvhyLFy+GXC43mi8kJAQhISGG\nx2lpaaYutcZRq9Xcj2aObWje2H7mT+o2VNy4Abvly2GzdSug0yHvX/9Czvjx0DZurJ+B76+HMkUb\nenh4lPmcSQKZs7Mz0tPTDY/T09Ph7OxsNM/evXsxffp0AEDz5s1RXFyM7OxsODo6mqJEIiIis6NI\nTNQHsa+/BgDkDRiAnLffhvaeIUFkHuQPn+XJeXp6Ijk5GampqdBoNIiNjYW/v7/RPGq1GmfOnAEA\nXL9+HcXFxXBwcDBFeURERGZF8fffcHz3Xbg98wxsvvkGeYMGIeXAAdxZsIBhzEyZpIdMoVBg5MiR\nmD9/PnQ6Hbp3746GDRti69at8PT0hL+/P15//XWsWbMGO3fuBACMGzcOMpnMFOURERGZBcXly7D/\n7DNY79gBWFggd9gw5IwdC129elKXRk9IJoQQUhfxJJKSkqQuwexJPfaBnhzb0Lyx/cxfVbeh8tIl\n2H36Kay/+w5CpULe0KH6IHb36gX05GrFGDIiIiJ6dMqLF/VBLDoawsoKuaNHI2fMGOhcXaUujSoZ\nAxkREVE1ozx3DvbLlsFq504Ia2vkjBuH3NGjoasOl9agKsFARkREVE0oz56F/bJlsP7pJ+js7JAz\nfrw+iN13ZQKqeRjIiIiIJGZx+jTsli6F9S+/QOfggOzJk5EzahQE71pTazCQERERScQiLg72S5fC\nKiYGOkdHZL33HnJHjoTgNThrHQYyIiIiE7M4flw/RmzvXujq1EHWlCnIHTECgtffrLUYyIiIiExE\ndfQo7JYsgdX+/dA6OSFr2jTkDh8OYWcndWkkMQYyIiKiKqY6dAj2S5bA8sABaF1ccGfmTOS9/jqE\nra3UpVE1wUBGRERUFYSAKjYW9kuXwvLgQWhdXXHnww+RN3QohI2N1NVRNcNARkREVJmEgOXvv8Nu\n6VJYHjkCrbs77syZg9xBgwBra6mro2qKgYyIiKgyCAHLX3+F/dKlUB0/Dm29erg9fz7yXnsNsLKS\nujqq5hjIiIiInoQQkP30E9Rz5kD155/Q1K+P2598grwBAwBLS6mrIzPBQEZERPSY5LduwWnsWFgc\nPAhNw4a4/Z//IK9/f0Clkro0MjMMZERERI/B4s8/4fzvf0N2+zY0y5cjtU8fwMJC6rLITMmlLoCI\niMjcWG/dCvWrr0IolUiLjoZu9GiGMXoi7CEjIiKqqKIiOH70EWy/+AKFzzyDzFWreONvqhQMZERE\nRBUgv3ULTqNHw/LIEeSMGYOsadMAJX+NUuXgO4mIiOghLE6cgPMbb0B2+zYyV65EfliY1CVRDcMx\nZEREROWw3rJFP17MwgJp0dEMY1Ql2ENGRERUmqIiOM6eDduNG1H47LPI+PxzCI4XoyrCQEZERHQf\neWqqfrzY0aPIGTsWWVOncrwYVSm+u4iIiO5x73ixjM8/R8HLL0tdEtUCHENGRER0l81XX+nHi6lU\nSPv+e4YxMhn2kBERERUVwfHDD2G7aRMKgoKQuXIlx4uRSTGQERFRrSZPSYHz6NFQHTvG8WIkGb7j\niIio1rI4fhzOo0dDducOx4uRpDiGjIiIaiWbL7/keDGqNthDRkREtUtRERxnzYJtVBQKunbVjxdz\ncpK6KqrlGMiIiKjWuHe8WPZbbyH7gw8AhULqsogYyIiIqHawOHZMP14sKwsZq1ah4KWXpC6JyIBj\nyIiIqMaziYqCul8/CCsrpP3wA8MYVTvsISMiopqrsFA/XuzLL1HQrRsyV6zgeDGqlhjIiIioRpLf\nvKkfL3b8OLLHj0f2lCkcL0bVFgMZERHVOBZHj+rHi+XkIGP1ahT06SN1SUTl4hgyIiKqUWw2bYK6\nf38IGxv9eDGGMTID7CEjIqKa4f7xYitXQtSpI3VVRBXCQEZERGZPfvMmnN94A6oTJzhejMwSAxkR\nEZk11dGjcCoZL7ZmDQp695a6JKJHZrJAFhcXh8jISOh0OgQHByMsLMzo+Q0bNuDs2bMAgKKiIty5\ncwcbNmwwVXlERGRuhIDNpk1w/PBDaOvXR/pXX0HTsqXUVRE9FpMEMp1Oh4iICMycORMuLi6YNm0a\n/P390aBBA8M8w4cPN/x/165duHLliilKIyIic1RYCMeZM2G7eTMKnnsOmcuXc7wYmTWTnGV56dIl\n1K1bF+7u7lAqlejcuTOOHj1a5vwHDhzAM888Y4rSiIjIzMiTk6F+9VXYbt6M7LffRsaGDQxjZPZM\n0kOWkZEBFxcXw2MXFxckJCSUOu+tW7eQmpoKHx+fUp+PiYlBTEwMACA8PBxqtbryC65llEol96OZ\nYxuaN7ZfxckOHIBy4EAgJwfFW7bA8pVXYCl1UWAb1gRSt2G1G9R/4MABdOzYEXJ56Z13ISEhCAkJ\nMTxOS0szVWk1llqt5n40c2xD88b2qwAhYLNxo368WIMGyPjqK2hatACqyX5jG5o/U7Shh4dHmc+Z\n5JCls7Mz0tPTDY/T09Ph7Oxc6ryxsbHo0qWLKcoiIiJzUFAAx/feQ53p01EYFIRbP/2kD2NENYhJ\nApmnpyeSk5ORmpoKjUaD2NhY+Pv7PzDfjRs3kJubi+bNm5uiLCIiqubkyclQ9+sH2y1bkD1hgn68\nmKOj1GURVTqTHLJUKBQYOXIk5s+fD51Oh+7du6Nhw4bYunUrPD09DeHswIED6Ny5M2QymSnKIiKi\nakx1+DCc3nwTsrw8ZKxdi4IXX5S6JKIqIxNCCKmLeBJJSUlSl2D2OPbB/LENzRvb7z5CwOaLL+A4\neza0DRsiY/16aKr5kRO2ofmTegxZtRvUT0REtVhBAepMnw6brVv11xdbsYKHKKlWYCAjIqJqQZ6U\npL8fZVwcsidNQva77wJlnHFPVNMwkBERkeRUhw/r70eZn4+MdetQ0LOn1CURmRT/9CAiIukIAZsN\nG+Dyr39B2Nsj7ccfGcaoVmIPGRERSaOgAHWmTYPN11+jICQEmZ99xvFiVGsxkBERkcnJb9zQjxc7\neZLjxYjAQEZERCamOnRIf32x/HxkRESgoEcPqUsikhz/HCEiItMQArbr18NlwAAIBwek7dzJMEZ0\nF3vIiIio6hUUoM7UqbD55hv9eLHlyyEcHKSuiqjaYCAjIqIqZTRe7J13kD15MseLEd2HgYyIiCqX\nTgflhQtQHT4My0OHYLl/P6DVImP9ehSEhkpdHVG1xEBGRERPRqOBxdmzUB08qA9hR45Afvs2AEBb\nty4KgoORM3EiNF5eEhdKVH0xkBER0aMpLITq5EmoDh2C6vBhqI4ehTw3FwCgadIE+T16oCgwEEUd\nO0LbsCEgk0lcMFH1x0BGRETlkuXlweL4cVgePqwPYX/+CVlBAQCguEUL5L/6Kgo7dkRRhw7Q1asn\ncbVE5omBjIiIjMju3IHq6FHDGDCLU6cg02gg5HIUt26N3KFDUVQSwJydpS6XqEZgICMiquXk6en6\nQ4+HDsHy0CEo4+MhEwLCwgLFfn7IGTNGfwjS35+XqiCqIgxkRES1jDwp6Z/Dj4cPwyIhAQAgrKxQ\n1L49ciZPRmHHjihu3x7C2lriaolqBwYyIqKaTAgorl41HH5UHT4M5dWrAACdnR2KOnRAfr9+KAwM\nRLGfH6BSSVwwUe3EQEZEVJMIAeXFi4beL8vDh6G4eRMAoHVyQlFgIHKHD0dRx44o9vYGlPw1QFQd\n8JNIRGTOtFpYxMf/cwmKw4ehyMjQP+Xurj/78e4lKDTNmvEK+UTVFAMZEZE5KSqCxcmT+jFgJdcA\ny84GAGgaNUJhcLAhhGmbNOE1wIjMBAMZEVE1JsvPh8WJE/9cguL4cchLrgHm5YX8l19GUceOKOzQ\nAbr69SWulogeFwMZEVE1IsvOhurYMcMlKCxOnoSsuBhCJoPG2xt5gwfrD0EGBkKnVktdLhFVEgYy\nIiIp5edDFh0Nh9279ZegOHMGMp0OQqlEcZs2yHnjDX0ACwiAcHSUuloiqiIMZEREUtDpYL19O+z/\n8x8ok5KgtLTUXwNswgT9JSiefhrC1lbqKonIRBjIiIhMTPX773CYNw+qM2dQ5OuL4tWrccvHB7C0\nlLo0IpIIAxkRkYkoz5+Hw/z5sNq7F5oGDZC5YgXyX34Zajc3IC1N6vKISEIMZEREVUyekgL7RYtg\ns2ULhJ0d7sycidwRIwArK6lLI6JqgoGMiKiKyHJzYbd6NWxXrYJMo0HuyJHInjgRwtlZ6tKIqJph\nICMiqmwaDWy2boX9okVQpKYiv3dvZE2bpr9QKxFRKRjIiIgqixCw3LsXDvPnw+LCBRT5+yNj7VoU\n+/tLXRkRVXMMZERElUB55gwc58yB5YED0DRpgoz//hcFL77IWxcRUYUwkBERPQHFjRuwX7AA1jt2\nQFenDu7MmYPcoUMBlUrq0ojIjDCQERE9BllWFuxWroTd2rUAgJxx45Dz1lu8mj4RPRYGMiKiR1Fc\nDJuoKNgvWQJFRgby+vZF9gcfQNuggdSVEZEZYyAjIqoIIWD1889wmD8fyitXUNi5MzJmzUKxr6/U\nlRFRDcBARkT0EBYnTsBhzhxYHj2K4mbNkP7FFygMDuaAfSKqNAxkRERlUFy9CodPPoH1Dz9A6+qK\n2+HhyBs4EFDyq5OIKpfJvlXi4uIQGRkJnU6H4OBghIWFPTBPbGwsvvnmG8hkMjRu3BgTJ040VXlE\nRAayzEzYf/opbDdsgFAqkT1pEnLGjoWws5O6NCKqoUwSyHQ6HSIiIjBz5ky4uLhg2rRp8Pf3R4N7\nBsEmJyfju+++w9y5c2FnZ4c7d+6YojQion8UFsI2MhL2n30GWVYW8l57DdnvvQdd3bpSV0ZENZxJ\nAtmlS5dQt25duLu7AwA6d+6Mo0ePGgWyPXv2IDQ0FHZ3/wJ15KnjRGQqOh2sv/8e9uHhUCYmoqB7\nd2TNmAFNq1ZSV0ZEtYRJAllGRgZcXFwMj11cXJCQkGA0T1JSEgBg1qxZ0Ol06N+/P9q2bWuK8oio\nFlMdOgSHuXOhiotDsbc30r/6CoVBQVKXRUS1TLUZmarT6ZCcnIzZs2cjIyMDs2fPxqJFi2Bra2s0\nX0xMDGJiYgAA4eHhUKvVUpRboyiVSu5HM8c2fAwXLkA5YwbkP/wAUb8+NOvWQQwaBHuFAvYmLoXt\nZ/7YhuZP6jY0SSBzdnZGenq64XF6ejqcnZ0fmKdZs2ZQKpVwc3NDvXr1kJycDC8vL6P5QkJCEBIS\nYniclpZWtcXXAmq1mvvRzLENK06elgb7JUtgExUFYW2NrA8+QO4bb0BYWwOZmZLUxPYzf2xD82eK\nNvTw8CjzOXmVrvkuT09PJCcnIzU1FRqNBrGxsfD39zeap0OHDjh79iwAICsrC8nJyYYxZ0RET0qW\nnw+7zz6DW5cusImKQt6QIUg9cAA5EybowxgRkYRM0kOmUCgwcuRIzJ8/HzqdDt27d0fDhg2xdetW\neHp6wt/fH35+fjh58iQmT54MuVyOIUOGwN7e1AcOiKjG0WphvW0bHP7zHyhu3kR+aCiyp0+H5r7e\ndyIiKcmEEELqIp5EyckA9PjY1W7+2Ials9y/Hw5z58IiPh5Fbdsia9YsFHXsKHVZD2D7mT+2ofmT\n+pBltRnUT0RUWZTnzsFh/nxY7dsHTcOGyPj8cxT06QPITTJKg4jokVU4kJ05cwZubm5wc3NDZmYm\nvvzyS8jlcgwaNAh16tSpyhqJiCpEfvMm7Bctgs3WrRD29rgzaxZyR4wALC2lLo2IqFwV/nMxIiIC\n8rt/XW7cuBFarRYymQxr1qypsuKIiCpClpMD+4UL4fbMM7DZtg25o0Yh5cAB5I4ZwzBGRGahwj1k\nGRkZUKvV0Gq1OHnyJD7//HMolUq8+eabVVkfEVHZNBrYfPUV7BcvhuLWLeT36YOsqVOhbdJE6sqI\niB5JhQOZtbU1bt++jcTERDRo0ABWVlbQaDTQaDRVWR8R0YOEgGVMDBzmz4dFQgIKAwKQERGB4qef\nlroyIqLHUuFA1qNHD0ybNg0ajQbDhw8HAJw/fx7169evqtqIiB5gcfo0HObMgWVsLDRNmyJj3ToU\n9OgByGRSl0ZE9NgqHMjCwsLQoUMHyOVy1K1bF4D+6vpjxoypsuKIiEoorl+H/YIFsNmxA1onJ9ye\nNw95Q4YAFhZSl0ZE9MQe6bIX914/48yZM5DL5fD29q70ooiIAABCQJ6RAdvVq2EXEQHIZMgePx45\nb70F4eAgdXVERJWmwoFs9uzZGDhwIFq2bInvvvsOO3fuhFwuR2hoKPr27VuVNRJRTaPTQZ6eDnlq\nKhSpqfp/U1L++ffWLf30lBTICwoAAHmvvorsDz6AlsMkiKgGqnAgS0xMRPPmzQEAe/bswezZs2Fl\nZYVZs2YxkBGRXmEhFGlpkKekGAKVIXDd+++tW5BptQ+8XOfgAK2rK3Rubihq1w46Nzdo3dxQ+Oyz\n0Pj4SLBBRESmUeFAVnKHpZs3bwIAGjRoAADIzc2tgrKkZ71jB+zDw6FISoLWwwPZU6cin8GTaiMh\nIMvNNQ5XpQWtlBTIb99+8OUyGXRqNXSurtC6u0PTqhW0bm7QurtD5+oKnbs7tG5u0Lm58SbfRFRr\nVTiQtWjRAuvXr0dmZiYCAgIA6MNZTbwBuPWOHXCcMgXy/HwAgPLGDThOmQIADGVUc+h0kGdmPtib\ndevWP4cPSw4b3v0s3EuoVPog5eoKTdOmKAoM1D92d9f3cpUELbUaUPIubURE5anwt+Rbb72FH374\nAQ4ODnjppZcA6G/s/eKLL1ZZcVKxDw9/4BeQPD8f9uHhDGRU/RUVGcZgGYLWrVsP9HDJ09IgK+U6\ngjp7e0OPVZGfn/6wobu7/t97gpaoU4eXmiAiqiQVDmT29vYYNGiQ0bT27dtXekHVgSIp6ZGmE5ma\nLC8PFmfPwuLkSVjEx0OZlgbXGzf0YSsj44H5hUwGnYvLP4cNW7T4pzfrbvgyHDa0sZFgi4iIarcK\nBzKNRoMdO3Zg//79yMzMhJOTE4KCgtC3b18oa9jhCK2HB5Q3bpQ6ncjk8vNhER8Pi1OnoDp5Ehan\nTkGZkACZTgcA0Lq5AY0bQ9O4MXQBAaX2ZunUal6vi4ioGqtwkoqKisJff/2FN954A66urrh16xa2\nb9+OvLw8w5X7a4rsqVONxpABgM7aGtlTp0pYFdUKBQWwOH9e3/N1N4ApL140nJGodXVFsa8vCnr1\nQpGvL4p9faFzd4darUZmWprExRMR0eOqcCA7dOgQFi5caBjE7+HhgaZNm+L999+vcYGsZJwYz7Kk\nKlVUpA9fp07pf06ehMX584ZxXVpnZxT7+aHghRdQ7OeHojZtoKtXj+O2iIhqoEe+7EVtkd+3LwMY\nVZ7iYigvXIDq9GlD75fFuXOQFRUBAHR16qDI1xc5Y8ag2M8Pxb6++gugMnwREdUKFQ5knTp1woIF\nC9CvXz+o1WqkpaVh+/bt6NixY1XWR2R+NBooExL0hxxLer7i4yErLASgv/hpcZs2yPn3v1F897Cj\ntlEjhi8iolqswoFsyJAh2L59OyIiIpCZmQlnZ2d07twZ/fr1q8r6iKo3rRbKv/4yHHZUnTwJ5Zkz\nhtv96GxtUezri9zhw1Hs64siX19omzQB5HJp6yYiomql3EB25swZo8etW7dG69atIYSA7O5f8+fP\nn4cPb2lCtYFOB8Xly/per5Kf06chz8vTP21tjeI2bZA3ZIi+58vPD5qnnmL4IiKihyo3kK1atarU\n6SVhrCSYrVixovIrI5KSEFD8/bfxYcfTpyHPydE/bWWF4tatkTdgwD/hy8sLUCgkLpyIiMxRuYFs\n5cqVpqqDSDpCQJGY+M+lJkp6vu7c0T+tUqG4dWvk9+2LorsD7jXNm/N2QEREVGn4G4VqFyGgSErS\nh6+7vV6qkycNN8UWFhYobtUK+b176y814eenD18qlcSFExFRTcZARjWTEJBlZkKRlgbllStG1/pS\npKfrZ1EooGnZEvk9exoOOxa3bAlYWkpcPBER1TYMZGQ+hIAsJ0d/4+xbt/T3bbx1C/K7P4rU1H+e\nS0uDrLj4n5fK5dC0aIHC4GDDYcfiVq0Aa2sJN4iIiEiPgYykl58PRVraPwErNRXytDTjgFUy/e7l\nJO4l5HLoXF2hU6uhdXODpmVLw/0btW5u0Hp4QNO6NW+aTURE1RYDGVWN4mJ9qCotYN33WJ6dXeoi\ntM7O+qDl6ooif3/oXF31N8x2ddXfPFuths7NDTonJ57dSEREZo2BjCpOq4U8M7P8Q4UlhxIzM0td\nhM7BwRCqilu3NurJMgpaajVgYWHiDSQiIpIGA1l5hND/VJbKvDVOZS1LCCAzE8qEhDIPFRp6stLT\nIdNqH1iEzsoKOnd36NRqaJ56CrrAQOOerLv/16rVHLNFRERUCgayclj98gucR42SugyTcLvvsbCw\n+CdI1auHIj+/Mg8ZCltb3oeRiIjoCTCQlUPj5YWs996rnIVVYk+brDJ77YSAdb16yLa1/SeAubpC\n1KnDkEVERGQiDGTlyf62xwAAFZpJREFU0Hh5IWfyZKnLqHKWajXy09KkLoOIiKjW4l2PiYiIiCTG\nQEZEREQkMQYyIiIiIokxkBERERFJjIGMiIiISGImO8syLi4OkZGR0Ol0CA4ORlhYmNHzv/76KzZt\n2gRnZ2cAQI8ePRAcHGyq8oiIiIgkY5JAptPpEBERgZkzZ8LFxQXTpk2Dv78/GjRoYDRf586dMaqW\nXIiViIiIqIRJDlleunQJdevWhbu7O5RKJTp37oyjR4+aYtVERERE1Z5JesgyMjLg4uJieOzi4oKE\nhIQH5jt8+DDOnTuHevXqYdiwYVCr1aYoj4iIiEhS1eZK/U8//TS6dOkCCwsL7N69GytXrsTs2bMf\nmC8mJgYxMTEAgPDwcIa2JyD/6isoPvwQSExEvYYNoZ0zB7qBA6Uuix6DUqnkZ8GMsf3MH9vQ/End\nhiYJZM7OzkhPTzc8Tk9PNwzeL2Fvb2/4f3BwMKKiokpdVkhICEJCQgyP03jLn8divWMHHKdMgSw/\nXz/h2jXIx45FdnY28vv2lbY4emRqtZqfBTPG9jN/bEPzZ4o29PDwKPM5k4wh8/T0RHJyMlJTU6HR\naBAbGwt/f3+jeTIzMw3/P3bs2AMD/qly2YeHQ14Sxu6S5+fDPjxcooqIiIhqL5P0kCkUCowcORLz\n58+HTqdD9+7d0bBhQ2zduhWenp7w9/fHrl27cOzYMSgUCtjZ2WHcuHGmKK3WUiQlPdJ0IiIiqjoy\nIYSQuognkcQA8VjcOnSA8saNB6Zr6tdH6pEjElRET4KHS8wb28/8sQ3NX604ZEnVT/bUqdBZWxtN\n01lbI3vqVIkqIiIiqr2qzVmW9P/t3X9s1PXhx/HX9a6lH2w52jtbOIppqLBNI26knAx/4doYhhLw\nslXnj6WRJQ46NRtSzx8hyxQ9f2DVpQZiAPEPDRjOJiqoqWOg1m2Fjg1/MPk9bbvV6w84xvGj/dz+\nWHrfb8UqLfbed73nIzG5+9ynd6/yTsyrn8/73u/k6p+4nx8KydnWpj6fT9FgkAn9AAAYQCHLYLFA\nQLFAgEvtAAAYxi1LAAAAwyhkAAAAhlHIAAAADKOQAQAAGEYhAwAAMIxCBgAAYBiFDKOWFQ6ryO/X\nxJISFfn9ssJh05EAAPhKrEOGUckKh+WurU1soO5qbZW7tlaSWPwWAJByuEKGUSk/FEqUsX5ZsZjy\nQyFDiQAAGByFDKOSc5BN5wc7DgCASRQyjEp9Pt+QjgMAYBKFDKNSNBiUbVkDjtmWpWgwaCgRAACD\nY1I/RqX+ifv5oZCcbW3q8/kUDQaZ0A8ASEkUMoxasUCAAgYASAvcsgQAADCMQgYAAGAYhQwAAMAw\nChkAAIBhFDIAAADDKGQAAACGUcgAAAAMo5ABacwKh1Xk9ys7N1dFfr+scNh0JADAMLAwLJCmrHBY\n7tpaZcVikiRXa6vctbWSxIK4AJBmuEIGpKn8UChRxvplxWLKD4UMJQIADBeFDEhTzra2IR0HAKQu\nChmQpvp8viEdBwCkLgoZkKaiwaBsyxpwzLYsRYNBQ4kAAMPFpH4gTfVP3M8PheRsa1Ofz6doMMiE\nfgBIQxQyII3FAgHFAgF5vV5FIhHTcQAAw8QtSwAAAMMoZAAAAIZRyAAAAAyjkAEAABhGIQMAADCM\nQgYgZfVvnj6xpITN0wGMakkrZLt27dLdd9+tO++8Uw0NDYOe96c//UlVVVXav39/sqIBSEH9m6e7\nWlvliMcTm6dTygCMRkkpZLZta82aNbr//vtVV1en999/X59//vkZ58ViMW3ZskVTp05NRiwAKYzN\n0wFkkqQUsn379mnChAkqLi6Wy+XS7Nmz1dzcfMZ5GzZs0IIFC5SdnZ2MWABSGJunA8gkSSlkXV1d\n8ng8iecej0ddXV0Dzjlw4IAikYhmzJiRjEgAUhybpwPIJCmxdZJt23rxxRe1ZMmSbzy3sbFRjY2N\nkqRQKCSv1zvS8UY9l8vFv2OaG5VjuGKF4kuWyHH8eOJQfOxYacWKUfe7jsrxyzCMYfozPYaOeDwe\nH+kP+fTTT/XKK6/ogQcekCS9+uqrkqQbbrhBknT8+HHdeeedys3NlST19PQoLy9PtbW1Kisr+9r3\nbuP2xTljH8T0N1rH0AqHM2Lz9NE6fpmEMUx/yRhD39dc4U/KFbKysjK1t7ero6NDhYWFampq0l13\n3ZV4fezYsVqzZk3i+W9/+1vddttt31jGAIxu/ZunA8Bol5RC5nQ6dfvtt2vFihWybVvXXHONJk+e\nrA0bNqisrEzl5eXJiAEAAJCSknLLciRxy/Lccak9/TGG6Y3xS3+MYfozfcuSlfoBAAAMo5ABAAAY\nRiEDAAAwjEIGAABgGIUMAADAMAoZAACAYRQyADDECodV5PcrOzdXRX6/rHDYdCQAhqTEXpYAkGms\ncFju2lplxWKSJFdrq9y1tZLE7gRABuIKGQAYkB8KJcpYv6xYTPmhkKFEAEyikAGAAc5BdhkZ7DiA\n0Y1CBgAG9A2yhcpgxwGMbhQyADAgGgzKtqwBx2zLUjQYNJQIgElM6gcAA/on7ueHQnK2tanP51M0\nGGRCP5ChKGQAYEgsEFAsEJDX61UkEjEdB4BB3LIEAAAwjEIGAABgGIUMAADAMAoZAACAYRQyAAAA\nwyhkAAAAhlHIAAAjxgqHVeT3a2JJiYr8flnhsOlIQEpiHTIAwIiwwmG5a2sTm6i7Wlvlrq2VJBbA\nBb6EK2QAgBGRHwolyli/rFhM+aGQoURA6qKQAQBGhLOtbUjHgUxGIQMAjIg+n29Ix4FMRiEDAIyI\naDAo27IGHLMtS9Fg0FAiIHUxqR8AMCL6J+7nh0JytrWpz+dTNBhkQj/wFShkAIAREwsEKGDAWeCW\nJQAAgGEUMgAAAMMoZAAAAIZRyAAAAAyjkAEAABhGIQMAADCMQgYAwDBZ4bCK/H5l5+aqyO+XFQ6b\njoQ0xTpkAAAMgxUOy11bm9hA3dXaKndtrSSx9hqGjCtkAAAMQ34olChj/bJiMeWHQoYSIZ1RyAAA\nGAZnW9uQjgNfh0IGAMAw9Pl8QzoOfJ2kzSHbtWuX1q1bJ9u2VVFRoYULFw54/e2339Zbb72lrKws\n5ebm6o477lBJSUmy4gEAMCTRYHDAHDJJsi1L0WDQYCqkq6QUMtu2tWbNGj344IPyeDy67777VF5e\nPqBwXXHFFbr22mslSTt27ND69ev1wAMPJCMeAABD1j9xPz8UkrOtTX0+n6LBIBP6MSxJKWT79u3T\nhAkTVFxcLEmaPXu2mpubBxSysWPHJh6fOHFCDocjGdEAABi2WCCgWCAgr9erSCRiOg7SWFIKWVdX\nlzweT+K5x+PR3r17zzjvzTff1BtvvKHe3l4tX778K9+rsbFRjY2NkqRQKCSv1zsyoTOIy+Xi3zHN\nMYbpjfFLf4xh+jM9him1DtncuXM1d+5cvffee9q0aZN+9atfnXFOZWWlKisrE8/5i+Tc8Zdd+mMM\n0xvjl/4Yw/SXjDH0fc0XPpLyLcvCwkJ1dnYmnnd2dqqwsHDQ8/tvaQIAAGSCpBSysrIytbe3q6Oj\nQ729vWpqalJ5efmAc9rb2xOPW1paNHHixGREAwAAMC4ptyydTqduv/12rVixQrZt65prrtHkyZO1\nYcMGlZWVqby8XG+++aZ2794tp9OpvLw81dTUJCMaAACAcY54PB43HeJctLEi8jlj7kP6YwzTG+OX\n/kbrGFrhcMYs62F6DllKTeoHAACpgc3Tk4utkwAAwBnYPD25KGQAAOAMbJ6eXBQyAABwBjZPTy4K\nGQAAOEM0GJRtWQOOsXn6yGFSPwAAOAObpycXhQwAAHyl/s3TMfK4ZQkAAGAYhQwAAMAwChkAAIBh\nFDIAAADDKGQAAACGUcgAAEDGssJhFfn9ys7NVZHfLyscNpKDZS8AAEBGSqUN1LlCBgAAMlIqbaBO\nIQMAABkplTZQp5ABAICMlEobqFPIAABARkqlDdSZ1A8AADJSKm2gTiEDAAAZq38Dda/Xq0gkYiwH\ntywBAAAMo5ABAAAYRiEDAAAwjEIGAABgGIUMAADAMAoZAACAYRQyAAAAwyhkAAAAhlHIAAAADKOQ\nAQAAGOaIx+Nx0yEAAAAyGVfIoKCBXe3x7WIM0xvjl/4Yw/RnegwpZAAAAIZRyAAAAAyjkEGVlZWm\nI+AcMYbpjfFLf4xh+jM9hkzqBwAAMIwrZAAAAIa5TAeAGZFIRPX19erp6ZHD4VBlZaXmzZtnOhaG\nwbZtBYNBFRYWGv+WEIbuP//5j1atWqXPPvtMDodDixcv1rRp00zHwll6/fXX9Yc//EEOh0OTJ0/W\nkiVLlJOTYzoWvsFzzz2nlpYWud1urVy5UpJ07Ngx1dXV6YsvvtD555+vX//618rLy0taJgpZhnI6\nnbrttts0ZcoUxWIxBYNBTZ8+XSUlJaajYYg2b96sSZMmKRaLmY6CYVi3bp2+//3va+nSpert7dXJ\nkydNR8JZ6urq0pYtW1RXV6ecnBw99dRTampq0pw5c0xHwzeYM2eO5s6dq/r6+sSxhoYGXXLJJVq4\ncKEaGhrU0NCgW2+9NWmZuGWZoQoKCjRlyhRJkmVZmjRpkrq6ugynwlB1dnaqpaVFFRUVpqNgGI4f\nP65PPvlEP/rRjyRJLpdL5513nuFUGArbtnXq1Cn19fXp1KlTKigoMB0JZ+Giiy464+pXc3Ozrr76\naknS1Vdfrebm5qRm4goZ1NHRoYMHD+rCCy80HQVD9MILL+jWW2/l6lia6ujo0Lhx4/Tcc8/p8OHD\nmjJliqqrq5Wbm2s6Gs5CYWGh5s+fr8WLFysnJ0eXXnqpLr30UtOxMExHjhxJFOrx48fryJEjSf18\nrpBluBMnTmjlypWqrq7W2LFjTcfBEOzcuVNutztxpRPpp6+vTwcPHtS1116rxx9/XGPGjFFDQ4Pp\nWDhLx44dU3Nzs+rr67V69WqdOHFC27dvNx0L3wKHwyGHw5HUz6SQZbDe3l6tXLlSV155pS677DLT\ncTBE//jHP7Rjxw7V1NTo6aef1ocffqhnn33WdCwMgcfjkcfj0dSpUyVJs2bN0sGDBw2nwtnavXu3\nioqKNG7cOLlcLl122WX69NNPTcfCMLndbnV3d0uSuru7NW7cuKR+PrcsM1Q8HteqVas0adIkXX/9\n9abjYBhuvvlm3XzzzZKkjz76SK+99pruuusuw6kwFOPHj5fH41FbW5t8Pp92797NF2vSiNfr1d69\ne3Xy5Enl5ORo9+7dKisrMx0Lw1ReXq5t27Zp4cKF2rZtm2bOnJnUz2dh2Ay1Z88eLV++XBdccEHi\nsuzPfvYzzZgxw3AyDEd/IWPZi/Rz6NAhrVq1Sr29vSoqKtKSJUuS+lV7nJuNGzeqqalJTqdTpaWl\n+uUvf6ns7GzTsfANnn76aX388ceKRqNyu92qqqrSzJkzVVdXp0gkYmTZCwoZAACAYcwhAwAAMIxC\nBgAAYBiFDAAAwDAKGQAAgGEUMgAAAMMoZABGraqqKv3rX/8yHeMMGzduZBFfAAOwMCyApKipqVFP\nT4+ysv7v78A5c+Zo0aJFBlMBQGqgkAFImnvvvVfTp083HWNU6evrk9PpNB0DwDmikAEw7o9//KPe\neecdlZaWavv27SooKNCiRYt0ySWXSJK6urr0/PPPa8+ePcrLy9OCBQtUWVkpSbJtWw0NDdq6dauO\nHDmiiRMnatmyZfJ6vZKkv//973rkkUd09OhRXXHFFVq0aNFXbhq8ceNGff7558rJydFf/vIXeb1e\n1dTUJLbCqaqq0rPPPqsJEyZIkurr6+XxeHTTTTfpo48+0u9//3v9+Mc/1muvvaasrCz94he/kMvl\n0vr163X06FHNnz9fgUAg8XmnT59WXV2d/vrXv2rixIlavHixSktLE7/v2rVr9cknnyg3N1fXXXed\n5s2bl8j52WefKTs7Wzt37tTPf/5zVVRUjMzAAEga5pABSAl79+5VcXGx1qxZo6qqKj355JM6duyY\nJOmZZ56Rx+PR6tWrtXTpUr388sv68MMPJUmvv/663n//fd13331av369Fi9erDFjxiTet6WlRY8+\n+qiefPJJffDBB/rb3/42aIadO3dq9uzZeuGFF1ReXq61a9eedf6enh6dPn1aq1atUlVVlVavXq13\n331XoVBIv/vd77Rp0yZ1dHQkzt+xY4d++MMfau3atbr88sv1xBNPqLe3V7Zt67HHHlNpaalWr16t\n5cuXa/Pmzdq1a9eAn501a5bWrVunK6+88qwzAkhdFDIASfPEE0+ouro68V9jY2PiNbfbreuuu04u\nl0uzZ8+Wz+dTS0uLIpGI9uzZo1tuuUU5OTkqLS1VRUWFtm3bJkl65513dNNNN8nn88nhcKi0tFT5\n+fmJ9124cKHOO+88eb1eXXzxxTp06NCg+b773e9qxowZysrK0lVXXfW1536Z0+lUIBCQy+XS5Zdf\nrmg0qnnz5smyLE2ePFklJSUD3m/KlCmaNWuWXC6Xrr/+ep0+fVp79+7V/v37dfToUf3kJz+Ry+VS\ncXGxKioq1NTUlPjZadOmye/3KysrSzk5OWedEUDq4pYlgKRZtmzZoHPICgsLB9xKPP/889XV1aXu\n7m7l5eXJsqzEa16vV/v375ckdXZ2qri4eNDPHD9+fOLxmDFjdOLEiUHPdbvdicc5OTk6ffr0Wc/R\nys/PT3xhob8kffn9/v9nezyexOOsrCx5PB51d3dLkrq7u1VdXZ143bZtfe973/vKnwUwOlDIAKSE\nrq4uxePxRCmLRCIqLy9XQUGBjh07plgslihlkUhEhYWFkv5XTv7973/rggsuGNF8Y8aM0cmTJxPP\ne3p6zqkYdXZ2Jh7btq3Ozk4VFBTI6XSqqKiIZTGADMMtSwAp4ciRI9qyZYt6e3v1wQcfqLW1VT/4\nwQ/k9Xr1ne98Ry+99JJOnTqlw4cPa+vWrYm5UxUVFdqwYYPa29sVj8d1+PBhRaPRbz1faWmp3nvv\nPdm2rV27dunjjz8+p/c7cOCA/vznP6uvr0+bN29Wdna2pk6dqgsvvFCWZamhoUGnTp2Sbdv65z//\nqX379n1LvwmAVMQVMgBJ89hjjw1Yh2z69OlatmyZJGnq1Klqb2/XokWLNH78eP3mN79JzAW7++67\n9fzzz+uOO+5QXl6efvrTnyZuffbPv3r44YcVjUY1adIk3XPPPd969urqatXX1+utt97SzJkzNXPm\nzHN6v/LycjU1Nam+vl4TJkzQ0qVL5XL973/J9957r1588UXV1NSot7dXPp9PN95447fxawBIUY54\nPB43HQJAZutf9uKhhx4yHQUAjOCWJQAAgGEUMgAAAMO4ZQkAAGAYV8gAAAAMo5ABAAAYRiEDAAAw\njEIGAABgGIUMAADAMAoZAACAYf8FrXUhcxwpoosAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_mArAgg-BtyU",
        "colab_type": "code",
        "outputId": "6bc89835-bbf1-4b2d-f454-fb593eea7520",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "optimal_epoch(history)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Minimum validation loss reached in epoch 2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OcGgnNx25xxS",
        "colab_type": "text"
      },
      "source": [
        "Two things we observe from the graphs are:\n",
        "\n",
        "  - The training loss keeps decreasing after every epoch. Our model is learning to recognise the specific patterns in the training set.\n",
        "\n",
        "  - The validation loss keeps increasing after every epoch. Our model is not generalising well enough on the validation set.\n",
        "\n",
        "The training loss continues to go down and almost reaches zero at epoch 10. This is normal as the model is trained to fit the train data as good as possible.\n",
        "\n",
        "So, we are **overtraining**. \n",
        "\n",
        "We can try to do something about the overfitting. We are already using a drop out layer, which randomly removes certain features by setting them to zero. \n",
        "\n",
        "\n",
        "We can apply regularisation, which comes down to adding a cost to the loss function for large weights.\n",
        "\n",
        "There are L1 and L2 regularisation:\n",
        "\n",
        "  - L1 regularisation will add a cost with regards to the absolute value of the parameters. It will result in some of the weights to be equal to zero.\n",
        "\n",
        "  - L2 regularszation will add a cost with regards to the squared value of the parameters. This results in smaller weights.\n",
        "\n",
        "Let's try with **L2 regularisation.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "129xAPHMgmJy",
        "colab_type": "code",
        "outputId": "605d5661-132b-49eb-c999-bb7f5865e37a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "# Applying regularisation\n",
        "\n",
        "print('Building model...') \n",
        "\n",
        "reg_model = Sequential()\n",
        "reg_model.add(Embedding(2500, embed_dim, input_length = features.shape[1], dropout = 0.2))\n",
        "reg_model.add(LSTM(lstm_out, dropout_U = 0.2, dropout_W = 0.2))\n",
        "reg_model.add(Dense(2, kernel_regularizer=regularizers.l2(0.001), activation='softmax'))\n",
        "\n",
        "# Compile model\n",
        "reg_model.compile( optimizer='adam', # optimazer\n",
        "              loss = 'categorical_crossentropy', # loss function\n",
        "              metrics = ['accuracy']) # list of metrics\n",
        "\n",
        "reg_model.name = 'LSTM with Regularisation model'\n",
        "print(reg_model.summary())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Building model...\n",
            "Model: \"LSTM with Regularisation model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_2 (Embedding)      (None, 39, 128)           320000    \n",
            "_________________________________________________________________\n",
            "lstm_2 (LSTM)                (None, 200)               263200    \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 2)                 402       \n",
            "=================================================================\n",
            "Total params: 583,602\n",
            "Trainable params: 583,602\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ErZ7xdm_LfOK",
        "colab_type": "code",
        "outputId": "52bdcbf1-fb7c-4c0a-8c5d-4142c241d43e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 381
        }
      },
      "source": [
        "# Fit the model\n",
        "reg_history = reg_model.fit(X_train, Y_train, \n",
        "                    validation_split=0.33, \n",
        "                    batch_size = batch_size, \n",
        "                    nb_epoch = nb_epoch, verbose = True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 25633 samples, validate on 12626 samples\n",
            "Epoch 1/10\n",
            "25633/25633 [==============================] - 68s 3ms/step - loss: 0.5628 - acc: 0.7056 - val_loss: 0.5214 - val_acc: 0.7429\n",
            "Epoch 2/10\n",
            "25633/25633 [==============================] - 67s 3ms/step - loss: 0.4964 - acc: 0.7609 - val_loss: 0.5227 - val_acc: 0.7418\n",
            "Epoch 3/10\n",
            "25633/25633 [==============================] - 67s 3ms/step - loss: 0.4667 - acc: 0.7780 - val_loss: 0.5255 - val_acc: 0.7362\n",
            "Epoch 4/10\n",
            "25633/25633 [==============================] - 68s 3ms/step - loss: 0.4359 - acc: 0.7949 - val_loss: 0.5564 - val_acc: 0.7328\n",
            "Epoch 5/10\n",
            "25633/25633 [==============================] - 68s 3ms/step - loss: 0.4045 - acc: 0.8102 - val_loss: 0.5889 - val_acc: 0.7297\n",
            "Epoch 6/10\n",
            "25633/25633 [==============================] - 67s 3ms/step - loss: 0.3743 - acc: 0.8268 - val_loss: 0.6047 - val_acc: 0.7226\n",
            "Epoch 7/10\n",
            "25633/25633 [==============================] - 67s 3ms/step - loss: 0.3445 - acc: 0.8410 - val_loss: 0.6713 - val_acc: 0.7211\n",
            "Epoch 8/10\n",
            "25633/25633 [==============================] - 67s 3ms/step - loss: 0.3189 - acc: 0.8552 - val_loss: 0.7678 - val_acc: 0.7150\n",
            "Epoch 9/10\n",
            "25633/25633 [==============================] - 67s 3ms/step - loss: 0.2884 - acc: 0.8692 - val_loss: 0.8075 - val_acc: 0.7077\n",
            "Epoch 10/10\n",
            "25633/25633 [==============================] - 67s 3ms/step - loss: 0.2674 - acc: 0.8839 - val_loss: 0.8614 - val_acc: 0.7014\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z4H340HlO2Aw",
        "colab_type": "code",
        "outputId": "792bb15f-b1a4-4149-f80b-d8e34ff5cab9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        }
      },
      "source": [
        "eval_metric(reg_model, reg_history, 'loss')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAGHCAYAAAAeKU4NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd1gU19cH8O822KU3wQAKBBSxa0TE\nAoIY1FjQnyV2o1FjiSXFYCEajUpeS4yaqFGDsUVjLESNGhE7dkVjBRUVFVF6W8qy5/2DsHFpAgLL\nwvk8T564s3dmzuyduXO4c2dGQEQExhhjjDGmMUJNB8AYY4wxVttxQsYYY4wxpmGckDHGGGOMaRgn\nZIwxxhhjGsYJGWOMMcaYhnFCxhhjjDGmYZyQ1QCbNm2CWCzWdBjl8ujRIwgEApw5c6ZM89nb2+Pb\nb7+tpKiqTlVth0AgwNatW8u03lGjRsHHx+et133ixAkIBAI8ffr0rZf1JhUV89tISUlB3759YWxs\nDIFAgEePHmk0Hm1Smv2yKvenylaRbXdV7fvz5s2Dk5NTpa+nqpXnXFTR+2KtScji4+MxY8YMODs7\nQyqVwtLSEh4eHti8eTMUCoWmw3srgwYNwrNnz6pkXd9++y3s7e0rbHn16tVDTEwM3NzcyjTfpUuX\nMH369AqLo7aprN9PLBZj06ZNatPat2+PmJgYWFtbV/j6qqM1a9bg3LlzOHPmDGJiYlCvXr0KXf6b\nTohyuRwBAQFo0KABZDIZzMzM4OrqipUrVwLIS3oEAkGJ/wF5J3iBQIB+/foVWkdwcDAEAkGF/yFY\ncL90cnLCvHnzKmTZmzZtUtvGOnXqoGvXrjh//nyFLF/TfvjhB+zatavClnfmzJki/6D44osvasxv\nVt1oZ7dKGUVHR6Njx44Qi8WYP38+WrVqBYlEgrCwMCxduhTNmzdHy5YtNR1mmRERFAoFZDIZZDKZ\npsNRk52dDR0dnTeWE4lEqFu3bpmXX6dOnfKExf5Vlb+fjo5OuepYW0VGRqJJkyZo1qzZWy2ntMdQ\nQRMmTMDx48fxww8/oEWLFkhJScG1a9fw5MkTAHlJT25uLoC8trFt27YIDg5G27ZtCy2rfv36OHDg\nAGJjY2FlZaWavm7dOtjZ2VV4L1Vl75cikUgV88uXL7FgwQJ0794d9+7dg6WlZaWuu7Lk5ORAIpHA\n2Ni4StZnYGAAAwODKllXrUO1QM+ePcnKyoqSkpIKfZednU1paWmqf3/11VdkbW1NEomEXFxcaNu2\nbWrlAdDKlStp4MCBpKenR/Xq1aNdu3ZRUlISDRkyhAwMDMjBwYH++OMP1TxRUVEEgLZs2ULe3t4k\nlUrJwcGBfvvtN7Vlz5o1ixo1akQymYxsbW1p/PjxajEHBQWRSCSi0NBQatmyJUkkEvrrr79U0wuW\nO3PmDLVq1YpkMhm1bt2aLl68qLa+kJAQatq0Kenq6lKzZs3oxIkTqjiLEhQURADU/ps7dy4REdnZ\n2dHs2bNpwoQJZGZmRm3btiUiohUrVlCLFi1IX1+frKysaNCgQfT8+fNCv83p06fVPu/cuZM++OAD\nkslk5ODgQEFBQWqx2NnZ0YIFC9Q+BwQE0JQpU8jU1JQsLS1p2rRplJOToyqTkZFBY8eOJSMjIzIx\nMaEJEyaQv78/OTo6Frm9+d60DcePHycA9Pfff1OnTp1IJpORi4sL/fXXX2rLCQ8PJ3d3d9LR0SEn\nJyfauXNnoe14XXJyMslkskL74LNnz0gkEtHRo0eJiGjbtm3Utm1bMjIyInNzc+rRowfdu3dPbZ6C\n9VpwvfHx8ap92tLSkmbPnk0jRoygLl26qMr8/fff5OnpSaampmRkZEQeHh504cIFtWUW3D9e/32i\no6NVZc+dO0edOnUiqVRKJiYmNHjwYIqNjVV9P3fuXHJ0dKR9+/aRs7Mz6enpkaenJ0VERBRTS3lG\njhypFrNSqaQlS5aQg4MDSSQSevfdd+n7779Xm2ffvn3UsmVLkslkZGxsTK6urnT16lUiymsTpk+f\nTjY2NqSjo0N169alQYMGFbv+gr+Bp6cnERGlpKTQuHHjyMLCgnR0dOi9996jI0eOqObL3++3bt1K\n3bt3Jz09PZoxY0aR68j/bYpjbGxMq1atKvF3Krje/OPvdfm/ZadOnSgwMFA1/fHjxyQWi2nevHlq\n7U5BISEhJJFIKD09nYiI5HI56erqUocOHVRl/v77b5JIJJSamkpE6vulp6dnof0pKiqq1MdbQQXb\nSSKiGzduEAD6888/1ab/9ttv1KJFC9LV1SU7OzuaPn266jxBVLq2pOC+SES0ZcsWev20WzCmhIQE\nGjp0KNWrV4+kUik1bNiQli5dSkqlstByV65cSXZ2diQQCCgjI6PQ+m7evEnvv/8+GRsbk56eHjVq\n1Ig2b96s+r6kdi1/vyhqXy5q/9u0aRO5uLiQRCIhGxsbmj17tlrb6+npSWPGjKH58+eTlZUVmZqa\n0vDhw1X1XpzynG+JiO7evUs9evQgfX190tfXp549e1JkZKRamZ07d5KjoyPp6uqSu7s7BQcHFzoW\nIiMjqV+/fmRsbEwmJibUtWtXunHjhur7otq2t1HjE7L4+HgSCoXFnvRe98UXX5CZmRn9/vvvdO/e\nPVq4cCEJBAIKCQlRlQFAVlZWtGnTJoqMjKQJEyaQVCqlbt26UVBQEEVGRtLkyZNJT0+P4uLiiOi/\nnfudd96hrVu30t27d2n27NkkFApVDT8R0YIFC+jUqVMUFRVFISEh5OzsTCNGjFB9HxQURAKBgFxd\nXSk0NJQePHhAL1++LDIhEwgE1KlTJzp16hTduXOHunXrRvb29qqD5OnTpySTyWjMmDF069YtCgkJ\noVatWpWYkGVkZNBXX31Ftra2FBMTQzExMWoNqaGhIc2dO5fu3btHt27dIqK8g/7o0aP08OFDCgsL\nI3d3d/Lw8FAts7iEzMHBgXbu3EmRkZE0c+ZMEolEaklGUQmZiYkJLV68mCIiImjnzp0kFotpw4YN\nqjKffvopWVpaUnBwMN29e5f8/f3JyMioVAlZSduQf1A2b96cDh06RBERETRq1CgyNDSkhIQE1W9n\nbW1N3bt3p/DwcAoLC6M2bdqQTCYrcd8cPHgwdevWTW3ad999R7a2tpSbm0tERL/88gv9+eefdP/+\nfbp69Sr16tWLnJycKCsrSzXPmxIyPz8/cnR0pGPHjtHNmzdp6NChZGhoqNbA79mzh3bu3El3796l\nmzdv0pgxY8jU1FS1n798+ZJEIhGtWLFCtX+8/vvkN1oxMTFkaGhIgwcPphs3btDp06epWbNm1KlT\nJ9W65s6dS3p6euTr60uXL1+m8PBwat26NXXs2LHEuip4Ulq9ejVJpVJat24dRURE0Jo1a0hXV1e1\nX8TExJBEIqHvvvuOHj58SLdv36Zt27apGt1ly5aRjY0NHT9+nB4/fkwXL14slNC97uXLlzRw4EDq\n1KkTxcTEUHx8PBER9e/fn+zs7Ojw4cN0+/ZtmjJlCkkkErpz5w4R/bff29jY0NatW+nhw4f08OHD\nItfxpoSsUaNG9MEHH6jWXZLSJGRbtmwhJycnVVIQEBBAvr6+RSY4r8vIyCBdXV06fPgwEeUlaPkJ\naX5y4+/vT+3bt1fN8/p+GR8fT/b29vT555+r9ieFQlGq460oBeNNS0ujadOmEQBVjPnlTExMaPPm\nzfTgwQM6efIkNWvWjIYNG6YqU5q2pDwJWUxMDC1evJiuXLlCDx8+pC1btpC+vj798ssvass1NDQk\nPz8/Cg8Ppxs3bpBCoSi0vmbNmtHgwYPp1q1b9ODBA/rrr79o//79qu9LatcUCoUqQbl48aLavlxw\n/ztw4AAJhUJatGgR3bt3j3bs2EEmJiY0Z84cVRlPT08yNjamadOm0Z07d+jIkSNkamqqVqYo5Tnf\nZmRkUP369cnb25suX75Mly9fps6dO5Ojo6OqTbx69SoJhULy9/enu3fv0u7du8ne3l7tWHjx4gVZ\nWVnRJ598Qjdu3KC7d+/S5MmTyczMjF6+fElEnJCV2YULFwgA7d69u8Ry6enppKOjQz/++KPadD8/\nP/Ly8lJ9BkBTp05VfX758iUBoMmTJ6umJSQkEADVzp/f6BXc+dzd3dUO8oL27NlDOjo6qhNvfg/V\nqVOn1MoVlZABoCtXrqimnT9/ngDQ3bt3iSivN87Ozo4UCoWqzKFDh0pMyIjykkY7O7tC0+3s7Mjb\n27vY+fJdvXqVANDTp0+JqPiEbNmyZap5FAoFGRgY0Nq1a9XWVzAh69Wrl9q6unXrRh9++CER5TW+\nOjo6agkaEZGbm9sbE7I3bUP+Qfn6PvbixQu1hn79+vWkr6+vdsL4559/CECJCdmhQ4dIJBKpkhsi\noqZNm5K/v3+x88THxxMAOnPmjGpaSQlZZGSkqschX1ZWFllbWxc6obwuNzeXTExMaOvWrappIpGo\nUG9mwUZrzpw5ZGNjo5YwhoeHEwA6efIkEeU1+iKRSNXwERHt2LGDBAIByeXyYmMqeFKytbWlL7/8\nUq3MtGnTyMHBgYj+q8uoqKgilzdlyhTy8vJS66F4k4Ix5P++Bw8eVCvXqlUr+uijj4jov/1+/vz5\nb1z+mxKyM2fOUP369UkoFFKzZs1o7NixtHfv3iK3oTQJmVwuJzMzMwoNDSWFQkE2Nja0e/fuNyZk\nRHkn4vzff9asWTR69GhycXGhQ4cOERFR27Zt1drFgse1o6Ojqhc+X2mOt6Lkt4v5vSb5PT9ubm5q\nvTl2dna0Zs0atXlPnjxJACghIaHUbUl5ErKiTJkyhXx8fNSWa2xsXKh3qeD6jIyMCh2LJSnYrp0+\nfbrIY6Pg/texY0caMGCAWpkVK1aQVCpVHeOenp7UvHlztTKffPIJtWvXrsSYynO+3bBhA8lkMnr1\n6pWqzIsXL0gqldKvv/5KRERDhw5V+0OAiGjVqlVqx8LcuXPJzc1NrYxSqVTrZa/ohKzGD+qnUr47\n/f79+8jOzoaHh4fadE9PT9y6dUttWosWLVT/rlOnDkQiEZo3b66aZmpqCh0dHbx8+VJtPnd3d7XP\nHTp0UFv2nj174OHhAWtraxgYGGDo0KHIzs7Gixcv1OZzdXV94/YIBAK1OPMHVMfGxgIAbt++DVdX\nV4hEomLjK6uixqCcOHECvr6+qFevHgwNDdGxY0cAwOPHj0tc1utj+kQiESwtLVWxl2YeIG+b8+fJ\nr9927dqplSnNNpd2G15fv5WVFUQikdrv7eLiAlNTU1WZpk2bvnHcR9euXWFpaYnt27cDAK5evYqb\nN29ixIgRqjLh4eHo27cvHBwcYGhoiPr16xcZX3Fu374NIG/wfT4dHZ1C+1lUVBSGDx8OJycnGBkZ\nwcjICMnJyaVeT75bt26hXbt2auOjWrRoAWNjY7XjwdraWm1MkbW1NYio0HFVnJSUFDx9+rTIY/rR\no0fIyMhA8+bN4evri6ZNm6Jv37744YcfEB0drSr70Ucf4Z9//oGTkxM++eQT7N69G9nZ2WXa3vzf\nt2AcHh4ehdqWoo6hsurQoQMePHiA06dPY+TIkYiNjUX//v3Ru3fvUreHr5NKpRg+fDjWr1+PgwcP\nQqFQoFevXqWa18vLC6GhoQCA0NBQdOnSRTUtJSUFV65cgbe3d5ljAko+3oojEokQHh6OK1euYMuW\nLXBwcMCWLVtUNye8evUKjx8/xmeffaYaK2VgYIDu3bsDyGtH3qYteROlUonAwEC0bNkSFhYWMDAw\nwNq1awsdYy4uLm8cx/XFF1/g448/RufOnTFv3jxcvXpV7fvyts0F3bp1q8hjLDMzEw8ePFBNe/18\nBKi3zyUp6/n21q1baNy4MSwsLFRlrKys4OzsrDrebt++rdbeAVBtf75Lly7hypUravuBoaEhHj16\nhMjIyDfGXR41PiFr0KABhEKhqlGsCBKJ5I3TBAIBlEplqZd54cIFDBgwAB4eHti7dy+uXr2KtWvX\nAoDaCUAkEkEqlb5xeUKhUC3Zyr9z6vWY8qdVFH19fbXPT548QY8ePWBvb48dO3bg8uXL+PPPPwHg\njSe1goOZS/N7lmaesm5zWbahqAHYZdkHiiISiTB06FBs3rwZALB582a4urrCxcUFAJCRkYH3338f\nAoEAQUFBuHjxIi5dugSBQFDmxOFNevbsiSdPnuDHH3/E+fPnER4eDktLywpfT76i6hN4+9/0dSKR\nCIcOHUJoaChcXV2xe/duNGzYEAcOHACQd9KPiorC0qVLoaOjg6lTp6Jly5ZISUmpsBheV/AYKi+x\nWIz27dvj888/R3BwMDZt2oQDBw7g1KlT5VreuHHjsGfPHixZsgQfffRRkW1gUby9vVU3FOQnX97e\n3ggNDcXJkychkUgKnRhLq7zHm5OTE5ydnTFs2DDMnDkTfn5+qn04f/4ffvgB4eHhqv+uX7+OyMhI\ntRs13tSWCIXCQglwTk5OifMsW7YMixcvxpQpU3D06FGEh4fj448/LnSMlWY/CQgIQEREBAYOHIib\nN2+iXbt2mDNnDoC3a5vLqzxtOlA559vSUCqV6NKli9p+EB4ejnv37lXYnb8F1fiEzMzMDN27d8fq\n1auRnJxc6PucnBykp6fDyckJurq6hRqskydPomnTphUSS8FbhcPCwtC4cWMAebcYW1hY4Ntvv4Wb\nmxsaNmxYqc/Zady4sdrdVkXFVxQdHR21eUpy6dIlyOVyrFixAh06dICzs3Op/iKqDE5OTtDR0cG5\nc+fUpr9pmytqGxo3bow7d+4gKSlJNe3WrVtF7pMFjRw5EtevX8e1a9fw22+/qfWO3blzB69evcLC\nhQvRuXNnuLi4IDExsUw9Ifn7YFhYmGpadnY2Ll26pPocHx+P27dvw9/fH76+vmjcuDGkUmmh3qrS\n7B9NmjTB+fPn1Rr+69evIzk5ucKONQAwMjKCra1tkce0g4MD9PT0AOQ15m3btsWsWbNw6tQpeHp6\nIigoSFXewMAAffv2xcqVK3H58mXcuXMHJ0+eLHUcTZo0AYBCcZw6dapCt7ck+Ql8aXsXC2rcuDFc\nXV1x9uxZfPzxx6Wez83NDVKpFPPnz0eDBg1Qt25deHl54fr169izZw/at28PXV3dYucvS3tTHqNH\nj0ZGRgZWr14NIK8npV69erh37x6cnJwK/SeVSkvdllhaWuL58+dq0wr2UhV06tQpdOvWDaNHj0ar\nVq3g5OT0Vr0x7777LiZOnIg//vgD8+fPx5o1awCUrl3LT6BKczwXdYzJZDI4OjqWO/byatKkCW7f\nvo24uDjVtNjYWNy7d091vDVu3FitvQOAs2fPqn1u06YNbt26BVtb20L7QWXdDVzjEzIA+OmnnyCR\nSPDee+9h+/btuH37Nu7fv4+tW7eiTZs2iIyMhJ6eHqZMmYKAgADs2rULERERWLRoEYKDgzFr1qwK\niWPjxo3Yvn07IiIi8PXXX+PcuXP47LPPAADOzs549eoVNm7ciIcPH2Lz5s346aefKmS9RZk4cSJi\nY2MxYcIE3LlzB8ePH8fs2bMBlPyXn4ODA168eIFz584hLi4OGRkZxZZt0KABBAIBli1bhqioKOzb\ntw/z58+v8G0pDX19fYwfPx5z5szBgQMHEBERgdmzZ+POnTslbm9FbcOQIUNgaGiIYcOG4fr16zh/\n/jxGjx5dqseVNG3aFK1atcLo0aORlJSEwYMHq76zs7ODrq4uVq1ahQcPHuDYsWOYOnVqmXoCnZyc\n0Lt3b0yaNAnHjx/H7du38fHHHyM1NVVVxtTUFHXq1MH69esRERGBc+fOYfDgwYXid3BwwPHjx/H8\n+XO1BvF1kydPRkpKCkaNGoWbN2/izJkzGD58ODp16oROnTqVOu7SmDlzJlatWoX169cjMjIS69at\nw5o1a1THdFhYGBYsWIALFy7gyZMnOHbsGG7cuKFKUpcsWYJt27bh1q1biIqKwi+//AKRSISGDRuW\nOgZHR0cMGDAAEydOxJEjR3D37l1MnToVN2/exJdfflmu7crOzi70l/uNGzcA5F0uWrt2LS5fvozH\njx/j2LFjmDhxIkxMTODl5VWu9QHAkSNHEBcXV6aTrI6ODjp06IBff/1VdWnSzMwMTZs2xdatW994\nudLBwQFnz57FkydPEBcXV+G9ICKRCNOmTcPixYtV+/vChQuxcuVKLFy4EDdv3sS9e/ewb98+jB8/\nHkDp2xIfHx/cvXsXP/74Ix48eID169fj999/LzEeZ2dnnDhxAsePH0dERATmzJmDCxculHm70tLS\nMGnSJISGhiIqKgrXrl3D4cOHVft1ado1Ozs7CIVC/PXXX3j58mWxfzzOnDkTu3fvRmBgICIiIvD7\n779j3rx5+Pzzz8v12Ja3NWTIENSpUweDBg3C1atXceXKFXz44YewsbHBoEGDAADTp0/HuXPnMHv2\nbERERGDv3r1YtmyZ2nImT56M3Nxc9OnTB6dPn8ajR49w5swZzJ49u1AyV1FqRUJWv359XL16FX5+\nfpg3bx5at26N9u3bY/369fjyyy9VWfPChQsxduxYTJs2TdVgbN26FV26dKmQOAIDA/Hzzz+jefPm\n2LJlC7Zu3YrWrVsDyLscNHv2bMyaNQvNmjXDjh07sGTJkgpZb1FsbGzw559/IiwsDC1btsTUqVOx\nYMECACjxkqifnx8GDBiADz74AHXq1MH//d//FVu2efPmWLVqFdatW4fGjRtj6dKlWLFiRYVvS2l9\n99136NWrF4YMGYK2bdsiMTERo0aNKnF7K2ob9PT08NdffyE+Ph5t27bF0KFDMX369FI/+2jkyJEI\nDw9Hjx49YG5urppuYWGBrVu34ujRo2jSpAm++OILLF26FEJh2Q7tX375BS1btkTPnj3h6ekJGxsb\n9O3bV/W9UCjErl278ODBAzRv3hyjRo3CtGnT8M4776gtZ9myZbhy5Qrs7e2L/SvSysoKf//9N54+\nfQpXV1f07NkTTZs2xR9//FGmmEtjwoQJmD9/PhYtWoTGjRvju+++Q2BgIMaMGQMAMDY2xrlz59Cn\nTx80aNAAo0ePxtChQxEQEAAgr5dt+fLlcHd3R7NmzbB3717s3r0bzs7OZYpjw4YN8PX1xbBhw9Ci\nRQucPXsWBw4cQKNGjcq1XdHR0WjVqpXaf/njz7p3745t27ahR48ecHZ2xkcffYQGDRrg7NmzauNq\nykpPTw9mZmZlns/LywsKhUIt+fL29i40rSjffPMNkpKS4OzsjDp16qiepVaRxowZg9zcXNUJefjw\n4fj9999x4MABtG3bFq6urpg3bx5sbGxU85SmLfHx8cG3336LRYsWoUWLFggNDcXXX39dYiwBAQHw\n9PREnz594O7ujsTEREyZMqXM2yQWi5GYmIgxY8bAxcUFvr6+sLKyUo1FLU27ZmVlhcWLFyMwMBDv\nvPMO+vTpU+S6evTogV9++QW//vormjZtiunTp2PixImYO3dumeOuCDKZDH///Td0dXXh4eEBT09P\n6Ovr4/Dhw6oEMb9zZseOHWjWrBkCAwPx/fffqy3HysoK586dg4WFBfr16wdnZ2cMHToUjx8/LtTu\nVRQBlWeUJyuTR48ewcHBAadPny40cLA6yb9cc+PGjbd+qKW28Pb2hqmpKXbv3q3pUBhjWozbEva2\nasWT+lnR1qxZgxYtWsDa2hq3b9/G9OnT4ebmVmOTsX/++QdXr16Fu7s7srOzsWXLFhw/fhyHDh3S\ndGiMMS3CbQmrDJyQ1WKPHz/G4sWLERsbi7p166Jr16747rvvNB1WpREIBFizZg2mTJkCpVKJRo0a\nYe/evejWrZumQ2OMaRFuS1hl4EuWjDHGGGMaVisG9TPGGGOMVWeckDHGGGOMaRgnZIwxxhhjGqb1\ng/oLPgmZlZ2FhUWxD/Fk2oHrULtx/Wk/rkPtVxV1mP9e6aJwDxljjDHGmIZxQsYYY4wxpmGckDHG\nGGOMaZjWjyEriIiQmZkJpVJZphcs12axsbHIysqq8vUSEYRCIaRSKdcVY4yxWq3GJWSZmZmQSCQQ\ni2vcplUasVgMkUikkXUrFApkZmZCJpNpZP2MMcZYdVDjLlkqlUpOxrSIWCyGUqnUdBiMMcaYRtW4\nhIwvfWkfrjPGGGO1HXclVbCEhAQMGjQIAPDq1SuIRCKYmZkBAA4ePAgdHZ03LmP69OmYNGkSnJyc\nSrXO7du34+7du5g/f375A2eMMcaYxtT6hEy2Zw8MAwMhev4cudbWSPX3h7xfv3Ivz8zMDEePHgUA\nLFu2DPr6+vjkk0/UyhCRakB7Ub7//vtyr58xxhhj2qfGXbIsC9mePTCeMQPiZ88gIIL42TMYz5gB\n2Z49Fb6uqKgodO7cGZMnT4aXlxdiY2MxY8YMdO/eHV5eXmpJmJ+fH27evAmFQgEXFxcsWrQIPj4+\n6NWr1xufIvzkyRP0798fPj4++PDDD1VvMggODoa3tzd8fHzQv39/AMCdO3fQo0cP1fTHjx9X+HYz\nxhhj7M1qdUJmGBgIoVyuNk0ol8MwMLBS1nf//n2MHTsWJ06cwDvvvIOZM2fi0KFDOHr0KE6dOoWI\niIhC86SkpKBdu3YICQnBe++9hx07dpS4jlmzZmHIkCEICQlBz549MXfuXADA8uXLsXPnToSEhGDj\nxo0AgF9//RXjx49HaGgoDh48CCsrq4rfaMYYY4y9Ua1OyETFvAezuOlvy87ODi1atFB9Dg4Ohq+v\nL7p164bIyMgiEzKpVApvb28AQPPmzREdHV3iOq5du4Y+ffoAAPr374+LFy8CAFxdXTF16lRs375d\ndVdjmzZtsHLlSqxevRrPnz+HVCqtkO1kjDHGtAYRdE6dAp4902gYtTohyy3mJZ/FTX9benp6qn8/\nfPgQGzZswO+//46QkBB4eXkV+XDW128CEIlEyM3NLde6lyxZgs8//xzR0dHo1q0bkpKS0L9/f2zY\nsAE6OjoYNmwYzp8/X65lM8YYY1pHqYT08GFY9OwJi8GDIVq3TqPh1OqELNXfH8oCDyRVymRI9fev\n9HWnpaXBwMAAhoaGiI2NxYkTJypkua1bt8b+/fsBAHv27IGbmxsA4PHjx3jvvfcwY8YMGBsb48WL\nF3j8+DEcHBwwbtw4dOnSBSS8rpkAACAASURBVHfu3KmQGBhjjLFqS6GAbPdu1OnSBWZjxkCYmIik\n//s/5M6erdGwavVdlvl3U1bkXZal1axZMzRo0AAeHh6wtbWFq6trhSx34cKF+Oyzz7B69WpYWFhg\n+fLlAIB58+YhOjoaRAQPDw80atQIK1asQHBwMCQSCaysrPD5559XSAyMMcZYtZOZCb1du2Dw008Q\nP3mCHGdnJK5eDXmvXoBYDD1dXSA1VWPhCYiINLb2CvC8wHivjIwMtUuD7M3EYjEUCoXG1s919vYs\nLCzeeAcuq764/rQf12H1JUhPh96WLTD4+WeIYmOR3aoVUqdMQZaPD/Da46eqog6tSxgSVat7yBhj\njDFWMwkSE6G/aRMMNmyAMCkJWR06IPGHH5DdsSNQDd8QwwkZY4wxxmoMYWwsDNavh97mzRCmp0P+\n/vtImzwZOe+9p+nQSsQJGWOMMca0nig6GgY//QS9nTuBnBzIe/dG2qRJUDRurOnQSoUTMsYYY4xp\nLXFkJAxWr4Zs715AKETGwIFImzABuQ4Omg6tTDghY4wxxpjWkdy4AYNVqyA9dAgklSJ99GikjRsH\nZSU9S7SycULGGGOMMe1ABJ0LF/ISsRMnoDQyQtqUKUgfMwZKc3NNR/dWavWDYStD//79Cz3kdf36\n9fB/w8NmGzRoAAB48eIFxo4dW+yyr1+/XuJy1q9fD/lr7+ccPnw4kpOTSxF5yZYtW4a1a9e+9XIY\nY4yxMiOC7rFjMO/bFxb/+x8k//yDlJkzEXvxIlJnzND6ZAzghKzC+fn5ITg4WG1acHAw/Pz8SjV/\n3bp1sX79+nKvf8OGDWoJ2ZYtW2BsbFzu5THGGGMak5sL6f79qOPrC/MRIyB69gxJ336LlxcuIG3y\nZJChoaYjrDCckFWwDz74AMeOHUN2djYAIDo6GrGxsXBzc0N6ejoGDhwIX19fdOnSBUeOHCk0f3R0\ntOpl4nK5HBMmTICnpyfGjBmDzMxMVTl/f390794dXl5eWLp0KQBg48aNiI2NxYABA9C/f38AgJub\nGxISEgAA69atg7e3N7y9vVVJX3R0NDp27Igvv/wSXl5eGDx4sFpCV5SbN2+iZ8+e8PHxwZgxY5CU\nlKRaf+fOneHj44MJEyYAAM6dO4euXbuia9eueP/995GWllbu35YxxlgtkZMD2c6dsOzcGWaffAKB\nXI7E5cvx8uxZZHz0EajAaw9rgho9hszo668huX27QpeZ07gxUubPL/Z7U1NTtGzZEsePH4evry+C\ng4PRq1cvCAQC6OrqYuPGjTA0NERCQgJ69eqF999/H4JiHlC3efNmyGQynDx5Erdv30a3bt1U3331\n1VcwNTVFbm4uBg0ahNu3b2PMmDH4+eefsWvXLpiZmakt68aNG/j9999x4MABEBF69uwJd3d3GBsb\n4+HDh1i9ejWWLFmC8ePH46+//sL//ve/Yrdx2rRpWLBgAdzd3bFkyRIsX74c8+fPx48//ohz585B\nV1dXdZl07dq1WLRoEVxdXZGeng5dXd2y/NyMMcZqE7kcejt2wGDNGoifPUNOkyZIWLsWmT16ACKR\npqOrVNxDVglev2z5+uVKIkJgYCB8fHwwaNAgvHjxAq9evSp2ORcuXEC/f9+r2bhxY7i4uKi+279/\nP3x9feHr64t79+4hMjKyxJguXryIbt26QU9PD/r6+ujevTsuXLgAAKhfvz6aNm0KAGjevDmio6OL\nXU5KSgqSk5Ph7u4OABgwYIBqOS4uLpg8eTJ2794NsTgv13d1dcU333yDjRs3Ijk5WTWdMcYYyydI\nSYHB6tWwatcOJnPmINfaGvFbtuDVkSPI7NWrxidjQA3vISupJ6sy+fr6Yt68efjnn38gl8vRvHlz\nAMCePXsQHx+PQ4cOQSKRwM3NDVlZWWVe/pMnT7Bu3TocPHgQJiYmmDZtmtrlzLLS0dFR/VskEpV7\nWZs3b8b58+dx9OhRrFy5EseOHcPkyZPRpUsXhIaGws/PD9u3b4eTk1O5Y2WMMVZzCBMSoL9hA/SD\ngiBMSUFm585I/PRTZLdrp+nQqhz3kFUCfX19tG/fHp999pnaYP7U1FRYWFhAIpHg7NmzePr0aYnL\ncXNzw759+wAAd+/exZ07d1TLkclkMDIywqtXr3D8+HHVPAYGBkWO03Jzc8ORI0cgl8uRkZGBw4cP\nw83NrczbZmRkBGNjY1Wv2O7du9GuXTsolUo8f/4cHTp0wOzZs5Gamor09HQ8evQILi4umDRpElq0\naIH79++XeZ2MMcZqFuHz5zCaOxeWbdvCYOVKZHXsiFeHDiFh27ZamYwBNbyHTJP8/PwwZswYrFmz\nRjWtX79+GDlyJLp06YLmzZu/sadoxIgR+Oyzz+Dp6YkGDRqoetqaNGmCpk2bwsPDA9bW1nB1dVXN\nM3ToUAwdOhRWVlb4448/VNObNWuGAQMG4IMPPgAADB48GE2bNi3x8mRxVqxYAX9/f2RmZqJ+/fpY\nvnw5cnNz8emnnyI1NRVEhNGjR8PY2BhLlixBWFgYhEIhGjZsCC8vrzKvjzHGWM0gioqCwZo10Pv9\nd0CphLxvX6RNngzFv49+qs0ERESaDuJtPH/+XO1zRkYG9PT0NBSNdhKLxVAoFBpbP9fZ27OwsEBc\nXJymw2DlxPWn/bgOSya+cyfv9UZ//glIJMj48MO81xvVq6fp0FSqog6tS3iLAPeQMcYYY6xSSK5c\ngeGqVZAePQqlvj7Sx4/Pe72RpaWmQ6t2OCFjjDHGWMUhgs6ZMzBctQq6Z89CaWKClC++QPqoUSBT\nU01HV21xQsYYY4yxt6dUQjckBIYrV0Ln2jXkWlkh+euvkTFsGEhfX9PRVXs1LiHT8iFxtRLXGWOM\naTGFArL9+2GwejUkd+9CUb8+kgIDkTFgACCVajo6rVHjEjKhUAiFQsEPINUSCoUCQiE/fYUxxrRO\nVhb0du2CwU8/Qfz4MXIaNkTiqlWQ9+4N8Dm4zGrcLyaVSpGZmYmsrKxiX0nE1Onq6pbrAbVvi4gg\nFAoh5b+gGGNMawgyMqC3dSsM1q2D6MULZLdsiYSvv0bm++8D/Ad2udW4hEwgEEBWA186Wpn4dm3G\nGGNvIkhKgn5QEPQ3boQoMRFZ7dsj8fvvkd2pE8AdIG+txiVkjDHGGKs4oidPoLd1K/R//RXCtDRk\n+vgg4dNPkdOmjaZDq1E4IWOMMcaYGmFcHKT790Nv717oXLkCEgoh79ULaZMmQdGkiabDq5GqLCEL\nDw9HUFAQlEolunTpovaORwCIi4vDjz/+iPT0dCiVSgwZMgStW7euqvAYY4yxWk2Qmgrp4cOQ7dsH\n3dOnIcjNRU7jxkiZPRvyPn2Qa2Oj6RBrtCpJyJRKJTZu3Ig5c+bA3NwcM2fORJs2bWBra6sqs3v3\nbri7u+P999/H06dPsXjxYk7IGGOMscqUlQXp8eOQ7d0LaUgIBJmZUNSvj7RJkyD384PC2VnTEdYa\nVZKQ3b9/H3Xr1oWVlRUAoH379rh06ZJaQiYQCJCRkQEg792Gpvw0X8YYY6zi5eZC59w5yPbtg+zg\nQQhTUpBrbo70wYMh79sXOa1b8yB9DaiShCwhIQHm5uaqz+bm5oiMjFQrM2DAAHz77bc4fPgwsrKy\nEBAQUOSyQkJCEBISAgAIDAyEhYVF5QVeS4jFYv4dtRzXoXbj+tN+1b4OiSC4ehXCHTsg3LULgpgY\nkIEBlH5+yBk0COTtDR2xGDqajlODNF2H1WZQ/9mzZ9G5c2f06tULERERWLVqFZYtW1booaE+Pj7w\n8fFRfebHNbw9fuyF9uM61G5cf9qvutah6MED6O3bB9nevRBHRYF0dJDp7Q353LnI9PEB8h8TlZSk\n2UCrgaqoQ2tr62K/q5KEzMzMDPHx8arP8fHxMDMzUysTGhqKWbNmAQAaNmyInJwcpKamwtjYuCpC\nZIwxxmoE4YsXkP35J2T79kHn+nWQQIDs9u3zxoV17w4yMdF0iKwIVZKQOTo6IiYmBi9fvoSZmRnC\nwsIwZcoUtTIWFha4efMmOnfujKdPnyInJwdGRkZVER5jjDGm1QRJSZAdOgTZ3r3QCQuDgAjZLVog\nee5cyHv3hrJuXU2HyN6gShIykUiE0aNHY+HChVAqlfDy8kK9evWwc+dOODo6ok2bNhgxYgTWrVuH\ngwcPAgAmTpzIrz5ijDHGiiOXQxoSAtm+fZCGhkKQnQ3Fu+8i9bPP8h5T4eio6QhZGQiIiDQdxNt4\n/vy5pkPQetV17AMrPa5D7cb1p/2qrA4VCuieOZP3mIpDhyBMT0du3bqQ9+6dd4dks2Z8h2Q51Yox\nZIwxxhgrJyJIrlzJe0zFn39CFB8PpbEx5H36QO7nh+x27QCRSNNRsrfECRljjDFWDYnv3YNs717I\n9u2DODoaJJUis2tXyPv2RWbnzoCurqZDZBWIEzLGGGOsmhA9fQpZcDBke/dCcucOSCRClocHUr/4\nApnduoEMDDQdIqsknJAxxhhjGiRMSIB0//68d0hevAgAyG7TBkkLFyKzZ08oq/MDZ1mF4YSMMcYY\nq2KC9HRIjxyBbO9e6J46BYFCgZyGDZHy1VeQ+/kht359TYfIqhgnZIwxxlhVyM6G7okTeY+pOHIE\nwsxMKGxskDZ+fN6LvF1c+A7JWowTMsYYY6yyKJXQuXAhb3D+wYMQJiUh19QU8oEDIe/bF9lt2gAF\nXhHIaidOyBhjjLGKRATxrVvQ27sXsuBgiGJioNTTQ2a3bpD7+SHLwwOQSDQdJatmOCFjjDHGKoAo\nKirvWWH79kFy/z5ILEaWlxeSAwKQ1bUrSE9P0yGyaowTMsYYY+wtSIODIQ4KgtWlSwCALHd3JI0d\nC3mPHiAzMw1Hx7QFJ2SMMcZYeSiVMAwMhOGPP0LZpAmSAwIg79ULShsbTUfGtBAnZIwxxlhZyeUw\nnToVsoMHkT5sGCTr1iE9KUnTUTEtxgkZY4wxVgbCV69g9tFHkISHI/nrr5E+bhwsxHw6ZW+H9yDG\nGGOslMT37sFsxAgI4+ORuGEDMrt103RIrIbgh58wxhhjpaB78iQs+vSBICcH8Xv2cDLGKhQnZIwx\nxtgb6G3ZArPhw5Fra4tX+/cjp3lzTYfEahi+ZMkYY4wVJzcXRgsXwmDdOmR6eyNxzRqQgYGmo2I1\nECdkjDHGWBEEGRkw+fRTyA4fRtro0UiZOxfgwfuskvCexRhjjBUgfPEi707KmzeRvGAB0keP1nRI\nrIbjhIwxxhh7jfjWLZiPHAlBcjISgoKQ5eOj6ZBYLcCD+hljjLF/6R47Bou+fQEixO3dy8kYqzKc\nkDHGGGMA9IKCYDZqFBTvvotXBw9C0bSppkNitQhfsmSMMVa75ebCaN48GPzyC+Tvv4+kH38E6elp\nOipWy3BCxhhjrNYSpKXBdOJESI8dQ9q4cUiZMwcQiTQdFquFOCFjjDFWKwmfP4f5yJEQ37uHpEWL\nkDFypKZDYrUYJ2SMMcZqHck//8Bs5EgI0tORsHkzsjp31nRIrJbjQf2MMcZqFemRIzDv2xckFiNu\n3z5Oxli1wAkZY4yx2oEI+j//DNMxY6BwdkbcgQNQuLhoOirGAPAlS8YYY7WBQgHjOXOgv2UL5D16\nIGnlSpBMpumoGFPhhIwxxliNJkhNheknn0B64gRSJ01Cqr8/IOQLRKx64YSMMcZYjSV6+hRmI0dC\nfP8+kpYsQcaQIZoOibEicULGGGOsRpJcuwazjz6CICsL8Vu2INvDQ9MhMVYs7rNljDFW40gPHoR5\n//4gmQxxf/7JyRir9jghY4wxVnMQweCnn2A2bhwUTZogbv9+KBo00HRUjL0RX7JkjDFWM+TkwHjW\nLOhv3w55795IXL4c4DspmZbghIwxxpjWEyQnw2zcOOieOYPUqVOR+sUXfCcl0yqckDHGGNNqoidP\nYDZiBMSPHiHx++8hHzhQ0yExVmackDHGGNNaksuXYTZ6NAS5uYjfvh3Z7dtrOiTGyoX7cxljjGkl\naXAwLAYOBBka4lVwMCdjTKtxQsYYY0y7EMHghx9gNnEislu0QNz+/ch1ctJ0VIy9Fb5kyRhjTHtk\nZ8Nkxgzo7dqFjH79kLR0KaCrq+moGHtrVZaQhYeHIygoCEqlEl26dIGfn5/a95s2bcKtW7cAANnZ\n2UhOTsamTZuqKjzGGGPVnCAxEWZjx0L33DmkfP450qZPBwQCTYfFWIWokoRMqVRi48aNmDNnDszN\nzTFz5ky0adMGtra2qjKjRo1S/fvQoUOIioqqitAYY4xpAVFUFMyHD4fo2TMkrloFeb9+mg6JsQpV\nJWPI7t+/j7p168LKygpisRjt27fHpUuXii1/9uxZdOzYsSpCY4wxVs3pXLiAOj17QpCUhPidOzkZ\nYzVSlfSQJSQkwNzcXPXZ3NwckZGRRZZ99eoVXr58iaZNmxb5fUhICEJCQgAAgYGBsLCwqPiAaxmx\nWMy/o5bjOtRuXH/FE27fDtH48YCdHRT79sGomg7e5zrUfpquw2o3qP/s2bNo164dhMU8YdnHxwc+\nPj6qz3FxcVUVWo1lYWHBv6OW4zrUblx/RSCC4fLlMFy+HFnu7khYvx5kYgJU09+J61D7VUUdWltb\nF/tdlVyyNDMzQ3x8vOpzfHw8zMzMiiwbFhaGDh06VEVYjDHGqqPMTJh8+ikMly9HxsCBiN++HWRq\nqumoGKtUVZKQOTo6IiYmBi9fvoRCoUBYWBjatGlTqNyzZ8+Qnp6Ohg0bVkVYjDHGqhlhfDzMP/wQ\nenv3IsXfH0nLlwM6OpoOi7FKVyWXLEUiEUaPHo2FCxdCqVTCy8sL9erVw86dO+Ho6KhKzs6ePYv2\n7dtDwLcxM8ZYrSO6fx/mI0ZA9OIFEtasQWbv3poOibEqIyAi0nQQb+P58+eaDkHr8dgH7cd1qN24\n/gCds2dhNnYsSCxGQlAQct57T9MhlQnXofarFWPIGGOMseLIdu6E+ZAhyLW0RNyBA1qXjDFWETgh\nY4wxphlKJQwXL4bpZ58h290dccHByK1fX9NRMaYR1e6xF4wxxmoBuRym06dDtn8/0ocORfLChYBE\noumoGNMYTsgYY4xVKWFcHMw++giSa9eQPGcO0j/5hN9JyWo9TsgYY4xVGXFEBMxGjIDw1Ssk/vwz\nMnv00HRIjFULnJAxxhirErqnTsF03DiQTIb43buR07KlpkNirNrgQf2MMcYqnd62bTAbNgy5trZ5\nd1JyMsaYGu4hY4wxVnmys2H03XcwWLsWmV5eSFyzBmRoqOmoGKt2uIeMMcZYpdA9cQJ1fHxgsHYt\n0keMQMKmTZyMMVYM7iFjjDFWoUSPH8Pom28gO3IECnt7xG/ejKwuXTQdFmPVGidkjDHGKoRALofB\n6tUwWLMGJBIhZeZMpI0dC+jqajo0xqo9TsgYY4y9HSJIDx6E0fz5ED97hgw/P6TMng1lCe/tY4yp\n44SMMcZYuYkjImAcEADdM2eQ4+KCuJUrkd2unabDYkzrcELGGGOszAQpKTBctgz6QUEgAwMkLVyI\njGHDADGfVhgrDz5yGGOMlZ5SCdmuXTBatAjC+HhkDBmCVH9/KM3MNB0ZY1qNEzLGGGOlIgkPh/Gc\nOdC5dg3Z772HhC1bkNO8uabDYqxG4ISMMcZYiYTx8TBcvBh6O3ZAaWGBxBUrIP/f/wAhP8qSsYrC\nCRljjLGiKRTQ//VXGC5dCkFGBtLHjUPq9On8cFfGKgEnZIwxxgrRCQuDcUAAJHfvItPDAynz50PR\noIGmw2KsxuKEjDHGmIrw2TMYL1gA2f79UNjaImHDBmR26wYIBJoOjbEajRMyxhhjQGYmDNatg8Gq\nVRAQIeXzz5E2YQIgk2k6MsZqBU7IGGOsltM9ehTG8+ZB/OgR5D16IOXrr5Fbr56mw2KsVuGEjDHG\nainRw4cwnjsX0tBQ5Dg5If6335Dl4aHpsBirlTghY4yxWkaQng6DlSth8PPPIB0dJH/9NdJHjwYk\nEk2HxlitxQkZY4zVFkSQBQfDaMECiF68QEb//nkvAbe01HRkjNV6nJAxxlgtIL59O+8l4OfPI7tZ\nMySsXYscV1dNh8UY+xcnZIwxVoMJEhNhtHQp9DZvhtLYGEnffYeMwYMBkUjToTHGXsMJGWOM1US5\nudD77TcYBgZCmJyMjBEjkPLFFyBTU01HxhgrAidkjDFWw0guX4ZxQAB0btxAlpsbkhcsgKJJE02H\nxRgrASdkjDFWQwhfvoTRwoXQ++MP5Nati8Qff4S8Tx9+yj5jWoATMsYY03Y5OdDfuBGG338PQVYW\nUidPRtqUKSB9fU1HxhgrJU7IGGNMi+meOgWjgABI7t9Hprc3kr/5BrnvvqvpsBhjZcQJGWOMaSFR\ndDSMvvkGskOHoLC3R/ymTcjq2lXTYTHGyokTMsYY0yZyOQx/+gkGP/0EEgiQ8tVXSBs3DpBKNR0Z\nY+wtcELGGGPagAjSQ4dg9M03ED99Cnnv3kieMwdKGxtNR8YYqwCckDHGWDUnvn8fRgEBkJ46hZxG\njRC3axey27fXdFiMsQrECRljjFVTgtRUGH7/PfQ3bgTp6yN5wQKkjxgBiLnpZqym4aOaMcaqG6US\nst27YbRoEYSvXiFj8GCk+vtDaW6u6cgYY5WEEzLGGKtGJP/8A+PZs6Fz5QqyW7VCQlAQclq21HRY\njLFKxgkZY4xVA8KEBBgGBkJv+3Yozc2RuHw55AMGAEKhpkNjjFUBTsgYY0yDBKmpEO7aBct58yBI\nTUX6xx8j9bPPQEZGmg6NMVaFqiwhCw8PR1BQEJRKJbp06QI/P79CZcLCwrBr1y4IBALY2dlh6tSp\nVRUeY4xVDiIIExMhevQI4keP8v4fFQXx48cQPXoEUXw8ACCrY8e8l4A3bKjhgBljmlAlCZlSqcTG\njRsxZ84cmJubY+bMmWjTpg1sbW1VZWJiYrBv3z4sWLAABgYGSE5OrorQGGPs7RFBGBurSrJeT7jE\njx5BmJLyX1GBALnvvINce3tkduuGXHt7yDp0QHzz5vwScMZqsSpJyO7fv4+6devCysoKANC+fXtc\nunRJLSE7duwYfH19YWBgAAAwNjauitAYY6x0cnMhev5clWSJHz2C6PFjVa+XUC5XFSWRCLn16kFh\nbw9569ZQ2NtDYWeHXAcHKOrVK/RUfamFBRAXV9VbxBirRqokIUtISID5a7drm5ubIzIyUq3M8+fP\nAQABAQFQKpUYMGAAWhZxZ1FISAhCQkIAAIGBgbCwsKjEyGsHsVjMv6OW4zqsINnZwKNHEDx8CMGD\nB3n//ftvREVBkJOjKkq6uoCDA8jREdS1KxTvvpv3b0dHoH59QCKBEIAQgOQNq+X6035ch9pP03VY\nbQb1K5VKxMTEYO7cuUhISMDcuXOxdOlS6Ovrq5Xz8fGBj4+P6nMc/1X51iwsLPh31HJch2Ugl0P8\n5Elez1ZUlHpP19OnECiVqqJKPT3k2ttD4eQERdeuyLWzy+vtsreH8p13ir8DsoxDLrj+tB/Xofar\nijq0trYu9rsqScjMzMwQ/+/AVQCIj4+HmZlZoTINGjSAWCyGpaUl3nnnHcTExMDJyakqQmSM1SCC\n1NS8JCs/4Xr0COJ/P4tevFArqzQxgcLeHtmtWyO3b18o7O3zLi3a20NpYcHjuhhjVaJKEjJHR0fE\nxMTg5cuXMDMzQ1hYGKZMmaJWpm3btjhz5gy8vLyQkpKCmJgY1ZgzxhhTU8o7F/Pl1qkDhb09sjp1\n+m8s17/jusjUVEMbwRhj/6mShEwkEmH06NFYuHAhlEolvLy8UK9ePezcuROOjo5o06YNWrRogevX\nr2P69OkQCoUYNmwYDA0NqyI8xlh1pVRCcvMmxHfu/Nfb9e/lxdfvXAQAhbX1f3cuvnZpMdfODvTv\nzUKMMVZdCYiINB3E28i/GYCVH4990H41qQ4FSUnQPXkS0uPHoXv8OET/btfrdy6qJVzF3LmoTWpS\n/dVWXIfar1aMIWOMsWIRQXz7NqShodANDYXOlSsQ5OZCaWKCzM6dkeXtnTe+y9YWkLzpfkXGGNNO\nnJAxxqqcIDUVuqdPQzc0FNLjx1UD7bObNUPa5MnI9PJCTuvWgEik4UgZY6xqcELGGKt8RBBHRuYl\nYMeOQefiRQgUCigNDZHl4YHMLl2Q1bkzlHwjD2OsluKEjDFWKQQZGdA5c0Z1KVL87BkAIMfFBWnj\nx+ddinzvPb4MyRhjKENCdvPmTVhaWsLS0hKJiYnYtm0bhEIhhgwZAhMTk8qMkTGmJUQPH6oSMN1z\n5yDIzoZSTw9ZnTohbcoUZHp5QWljo+kwGWOs2il1QrZx40bMnj0bALB582YAeY+zWLduHb766qvK\niY4xVr3J5dA9fz7vUmRoKMSPHgEAcpyckD5qFDK9vZHdti2gq6vZOBljrJordUKWkJAACwsL5Obm\n4vr16/jpp58gFosxfvz4yoyPMVbNiJ48USVgOmfPQpiZCaVUiuwOHZA2diyyvLyQa2en6TAZY0yr\nlDohk8lkSEpKQnR0NGxtbSGVSqFQKKBQKCozPsaYpmVlQefiRdWlSMn9+wAAhZ0dMoYMQZa3N7La\ntQNkMg0Hyhhj2qvUCVm3bt0wc+ZMKBQKjBo1CgBw9+5d2PB4EMZqHOGzZ3kPZg0Nhe6ZMxCmp4N0\ndJDl7o6M4cOR6eWF3Hff5fc8MsZYBSl1Qubn54e2bdtCKBSibt26APJeCP7JJ59UWnCaJNuzB4aB\ngRA9f45ca2uk+vtD3q+fpsNirHLk5EDnyhXVpUjJnTsAAIWNDeT9+uWNBevYEaSnp+FAGWOsZirT\nYy9ef+T/zZs3IRQK0bhx4woPStNke/bAeMYMCOVyAID42TMYz5gBAJyUsRpDGBsL3RMnID12DLqn\nT0OYkgISi5Hdti2SAwKQ5eUFRcOG3AvGGGNVoNQJ2dy5czF48GA0atQI+/btw8GDByEUCuHr64t+\nNSxJMQwMVCVj+YRyKARg4wAAG8pJREFUOQwDAzkhY9orNxeSq1dVlyJ1/vknb3LdupB/8EHeWLBO\nnUCGhhoOlDHGap9SJ2TR0dFo2LAhAODYsWOYO3cupFIpAgICalxCJirmheXFTWesuhLGx0P3xIm8\nS5EnTkCYlAQSCpHdpg1S/P2R6e0NRePG3AvGGGMaVuqEjIgAAC/+feecra0tACA9Pb0SwtKsXGtr\n1VPFC05nrFpTKiG5ceO/sWDh4RAQIdfCAplduyLTywtZnp4gfpgzY4xVK6VOyJydnfHLL78gMTER\nrq6uAPKSM8MaeHkj1d9fbQwZAChlMqT6+2swKsaKJkhKgvD4cZjs2wfd48chio8HCQTIadkSqZ9/\njixvb+Q0awYIhZoOlTHGWDFKnZBNmjQJ+/fvh5GREXr37g0AeP78OXr06FFpwWlK/jgxvsuSVUsK\nBSTXrkF68iR0T57M6wVTKiE0McnrAfP2RpanJ5Tm5pqOlDHGWCkJKP9apJZ6zuO63pqFhQXi4uI0\nHQYrgejxY+iePAndU6fynguWmgoSCpHTogWyPDwg7dcPrxwcAJFI06GycuBjUPtxHWq/qqhD6xKG\nPpW6h0yhUGDPnj04deoUEhMTYWpqCg8PD/Tr1w9icZmensEYewNBSgp0w8JUSVj+OyIVNjaQ9+6N\nLA8PZHXoADI1BQDoWlgAfDJgjDGtVepMauvWrXjw4AHGjh2LOnXq4NWrV9i9ezcyMjJUT+5njJWT\nQgHJ9et5PWAnT0Ln6lUIcnOh1NdHdvv2SPv4Y2R5ePDT8RljrIYqdUJ2/vx5LFmyRDWI39raGg4O\nDvjyyy85IWOsHETR0Xk9YCdPQvfsWQiTk/MG47dogbRJk5Dl6Yns1q0BHR1Nh8oYY6ySlfmxF4yx\n8hGkpUEnLEw1GF8cFQUAyH3nHch79Mi7DNmxI8jMTMORMsYYq2r/3969B0dZ3X8c/+wll8WEQHZJ\nYAlOhohVqdjSECGIQJM6FHWkGRpv6FDobxSoOi0SluLQTlvaRaVY21gYJiC2Uwc6bDOjBW1jFS+x\nEqC0IFIBkWpCTXMBgiy5Pfv7A3YlQpCEZM9u8n7NMD57nifPfpMz4CfnnH3OJQeyCRMmaMWKFZo5\nc2Zk4dvmzZs1fvz43qwPiF/t7UrYsycyCpa4c6dsbW2yXC61TJigT7/zHTVPnqy2nBymIQGgn7vk\nQDZr1ixt3rxZZWVlamxsVHp6uvLz8zVz5szerA+IK47q6sg6sKQ33pD92DFJUsv11+vkgw+q+eab\n1ZKbKyUlGa4UABBLLhrI9u7d2+H16NGjNXr0aIVCIdnO/ka/f/9+ffnLX+69CoEYZvv0UyW+/XYk\nhCUcPCjpzP6Qp2+5Rc2TJ6t50iSeCQYAuKiLBrLf/va3F2wPh7FwMPvNb37T85UBsciylLB372fT\nkDt2yNbaKis5WS0TJujUvfeemYa8+mqmIQEAl+yigay0tDRadQAxy15To6Q33ohMQzoaGiRJraNH\n6+T//d+Zachx46TkZMOVAgDiFU90BT7HFgyemYY8+1DWhPfflyS1Z2REtiVqnjRJ1pAhhisFAPQV\nBDLAsuTct0/J4Yeybt8uW0uLQsnJar7xRp26884z05DXXMM0JACgVxDI0C/ZP/nkzEL8s38cZ7cd\nar32Wn06Z86ZUbBx4ySXy3ClAID+gECG/iEYVNL27Z9NQ773niSp3eM580DW8DRkZqbhQgEA/RGB\nDGaFQlJbm2ytrVJzs2ytrR2O1doqW/i4pUW2lpYuHzuOHFHSO+/I1tysUGKiWvLydGLpUp2++Wa1\nXXedZLeb/ikAAPo5Alm8CoUky5La26X2dtnCx5b12fHnz4WPz722vV22K65QYm1tp4FGra1n2sLH\nFwpLnR1fSmjq4W25Qg6HQgkJUlKSQgkJsoYM0af3339mb8jx4xViGhIAEGMIZBfh3LdPA55/XrZw\nuDkbYMLH5wWhzq4Lvw5/zTnHkdeh0IXPnXt89ppIew/ydOHakNPZIfCEEhOls//tcOxyKTRwoEJJ\nSWfaEhK++Phz91Bi4mfvcfZcKCHhosdyOHr0ZwMAQG8jkF2Eo6ZGAwIBhez2M/+Tdzg6HMtmUyh8\nbLd3OO5wbUKCrOTk86+z2c5cd+7XnHuP8LXnngvf8yLnQufc+4Lnzt47XONAj0fHg8EOwebcMBQJ\nXuHAwxQfAAA9ikB2Ec2Fhfrvu++aLqPXhTwetZz9lCEAAIg+hjoAAAAMI5ABAAAYRiADAAAwjEAG\nAABgGIEMAADAsKh9ynL37t1av369LMtSQUGBZsyY0eH8a6+9pt/97ndKT0+XJE2bNk0FBQXRKg8A\nAMCYqAQyy7JUVlamxx57TG63W0uWLFFubq6ysrI6XJefn6+5c+dGoyQAAICYEZUpy4MHD2ro0KHK\nzMyU0+lUfn6+qqqqovHWAAAAMS8qgayhoUFutzvy2u12q6Gh4bzr3nnnHT366KNauXKl6nhQaa9z\nBQLKyMtTQnKyMvLy5AoETJcEAEC/FDNP6v/a176miRMnKiEhQX/9619VWlqqH/3oR+ddV1FRoYqK\nCkmS3++Xx9OVXRgRZn/+eTkWL5bt1ClJkrO6WoMWL1Zqaqqsu+82XB26yul08nchjtF/8Y8+jH+m\n+zAqgSw9PV319fWR1/X19ZHF+2GpqamR44KCAv3+97+/4L0KCwtVWFgYec1IWvdkLF0aCWNhtlOn\npKVLVfeNbxiqCt3l8Xj4uxDH6L/4Rx/Gv2j0odfr7fRcVKYsc3JydPToUdXW1qqtrU2VlZXKzc3t\ncE1jY2PkeMeOHect+EfPctTUdKkdAAD0nqiMkDkcDs2ZM0fLly+XZVmaOnWqRowYoY0bNyonJ0e5\nubnaunWrduzYIYfDoZSUFM2fPz8apfVb7V6vnNXVF2wHAADRZQuFQiHTRVyOGkZ0usUVCCitpET2\nYDDSZrlcOv744woWFRmsDN3BdEl8o//iH30Y/0xPWcbMon5EVzh0pfr9ctTUqN3rVZPPRxgDAMAA\nAlk/FiwqUrCoiN/sAAAwjL0sAQAADCOQAQAAGEYgAwAAMIxABgAAYBiBDAAAwDACGQAAgGEEMgAA\nAMMIZOizXIGAMvLyNCwrSxl5eXIFAqZLAgDggngwLPqkz28N5ayuVlpJiSSxGwEAIOYwQoY+KdXv\n77BPpyTZg0Gl+v2GKgIAoHMEMvRJjk42ne+sHQAAkwhk6JPavd4utQMAYBKBDH1Sk88ny+Xq0Ga5\nXGry+QxVBABA51jUjz4pvHA/1e+Xo6ZG7V6vmnw+FvQDAGISgQx9VrCoiAAGAIgLTFkCAAAYRiAD\nAAAwjEAGAABgGIEMAADAMAIZAACAYQQyAAAAwwhkAAAAhhHIgDjmCgSUkZenhORkZeTlyRUImC4J\nANANPBgWiFOuQEBpJSWyB4OSJGd1tdJKSiSJB+ICQJxhhAyIU6l+fySMhdmDQaX6/YYqAgB0F4EM\niFOOmpoutQMAYheBDIhT7V5vl9oBALGLQAbEqSafT5bL1aHNcrnU5PMZqggA0F0s6gfiVHjhfqrf\nL0dNjdq9XjX5fCzoB4A4RCAD4liwqEjBoiJ5PB7V1dWZLgcA0E1MWQIAABhGIAMAADCMQAYAAGAY\ngQwAAMAwAhkAAIBhBDIAAADDCGQAAACGEcgAAAAMI5ABiFmuQEAZeXkalpWljLw8uQIB0yUBQK+I\nWiDbvXu3HnnkET300EMqLy/v9Lq///3vKi4u1qFDh6JVGoAY5AoElFZSImd1tWyhkJzV1UorKSGU\nAeiTohLILMtSWVmZfvjDH2rVqlV666239PHHH593XTAY1NatWzVq1KholAUghqX6/bIHgx3a7MGg\nUv1+QxUBQO+JSiA7ePCghg4dqszMTDmdTuXn56uqquq86zZu3Kg77rhDCQkJ0SgLQAxz1NR0qR0A\n4llUAllDQ4PcbnfktdvtVkNDQ4drPvjgA9XV1Wns2LHRKAlAjGv3ervUDgDxzGm6AOnMlOZzzz2n\n+fPnf+G1FRUVqqiokCT5/X55PJ7eLq/Pczqd/BzjXJ/sw+XLFZo/X7ZTpyJNoQEDpOXL+9z32if7\nr5+hD+Of6T60hUKhUG+/yfvvv68//vGPWrp0qSTpT3/6kyTpW9/6liTp1KlTeuihh5ScnCxJOnbs\nmFJSUlRSUqKcnJyL3ruG6YvL5vF4VFdXZ7oMXIa+2oeuQECpfr8cNTVq93rV5PMpWFRkuqwe11f7\nrz+hD+NfNPrQe5ER/qiMkOXk5Ojo0aOqra1Venq6Kisr9fDDD0fODxgwQGVlZZHXP/7xj3Xfffd9\nYRgD0LcFi4r6ZAADgM+LSiBzOByaM2eOli9fLsuyNHXqVI0YMUIbN25UTk6OcnNzo1EGAABATIrK\nlGVvYsry8jHUHv/ow/hG/8U/+jD+mZ6y5En9AAAAhhHIAAAADCOQAQAAGEYgAwAAMIxABgAAYBiB\nDAAMcQUCysjLU0JysjLy8uQKBEyXBMCQmNg6CQD6G1cgoLSSEtmDQUmSs7paaSUlksTDcIF+iBEy\nADAg1e+PhLEwezCoVL/fUEUATCKQAYABjk4eat1ZO4C+jUAGAAa0d/LE7s7aAfRtBDIAMKDJ55Pl\ncnVos1wuNfl8hioCYBKL+gHAgPDC/VS/X46aGrV7vWry+VjQD/RTBDIAMCRYVKRgUREbUwNgyhIA\nAMA0AhkAAIBhBDIAAADDCGQAAACGEcgAAAAMI5ABAAAYRiADAAAwjEAGAOg1rkBAGXl5GpaVpYy8\nPLkCAdMlATGJB8MCAHqFKxBQWkmJ7MGgJMlZXa20khJJYkcC4HMYIQMA9IpUvz8SxsLswaBS/X5D\nFQGxi0AGAOgVjpqaLrUD/RmBDADQK9q93i61A/0ZgQwA0CuafD5ZLleHNsvlUpPPZ6giIHaxqB8A\n0CvCC/dT/X45amrU7vWqyedjQT9wAQQyAECvCRYVEcCAS8CUJQAAgGEEMgAAAMMIZAAAAIYRyAAA\nAAwjkAEA0E3hvToTkpPZqxOXhU9ZAgDQDezViZ7ECBkAAN3AXp3oSQQyAAC6gb060ZMIZAAAdAN7\ndaInEcgAAOgG9upET2JRPwAA3cBenehJBDIAALopvFenx+NRXV2d6XIQx5iyBAAAMCxqI2S7d+/W\n+vXrZVmWCgoKNGPGjA7n//KXv+jll1+W3W5XcnKyHnjgAWVlZUWrPAAAAGOiEsgsy1JZWZkee+wx\nud1uLVmyRLm5uR0C10033aRbbrlFkrRjxw5t2LBBS5cujUZ5AAAARkVlyvLgwYMaOnSoMjMz5XQ6\nlZ+fr6qqqg7XDBgwIHJ8+vRp2Wy2aJQGAABgXFRGyBoaGuR2uyOv3W63Dhw4cN51L730kv785z+r\nra1Ny5Yti0ZpAAAAxsXUpyynTZumadOm6c0339TmzZv1ve9977xrKioqVFFRIUny+/3yeDzRLrPP\ncTqd/BzjHH0Y3+i/+Ecfxj/TfRiVQJaenq76+vrI6/r6eqWnp3d6fX5+vtauXXvBc4WFhSosLIy8\n5mPGl4+Pa8c/+jC+0X/xjz6Mf9HoQ+9FdnGIyhqynJwcHT16VLW1tWpra1NlZaVyc3M7XHP06NHI\n8a5duzRs2LBolAYAADrhCgSUkZenYVlZysjLkysQMF1SnxWVETKHw6E5c+Zo+fLlsixLU6dO1YgR\nI7Rx40bl5OQoNzdXL730kvbs2SOHw6GUlBQtWLAgGqUBAIALcAUCSispkT0YlCQ5q6uVVlIiSexG\n0AtsoVAoZLqIy1FTU2O6hLjHUHv8ow/jG/0X//piH2bk5clZXX1ee9vw4ardvt1ARb2rX0xZAgCA\n+OLoZMCjs3ZcHgIZAAA4T3snozmdtePyEMgAAMB5mnw+WS5XhzbL5VKTz2eoor4tpp5DBgAAYkN4\n4X6q3y9HTY3avV41+Xws6O8lBDIAAHBBwaIiAliUMGUJAABgGIEMAADAMAIZAACAYQQyAADQb4W3\nh0pITja6PRSL+gEAQL8US9tDMUIGAAD6pVS/PxLGwuzBoFL9/qjXQiADAAD9UixtD0UgAwAA/VIs\nbQ9FIAMAAP1SLG0PxaJ+AADQL8XS9lAEMgAA0G+Ft4fyeDyqq6szVgdTlgAAAIYRyAAAAAwjkAEA\nABhGIAMAADCMQAYAAGAYgQwAAMAwAhkAAIBhBDIAAADDCGQAAACGEcgAAAAMs4VCoZDpIgAAAPoz\nRsggn4Fd7dGz6MP4Rv/FP/ow/pnuQwIZAACAYQQyAAAAwwhkUGFhoekScJnow/hG/8U/+jD+me5D\nFvUDAAAYxggZAACAYU7TBcCMuro6lZaW6tixY7LZbCosLNT06dNNl4VusCxLPp9P6enpxj8lhK77\n9NNPtXr1an300Uey2WyaN2+err76atNl4RK9+OKL+tvf/iabzaYRI0Zo/vz5SkxMNF0WvsAzzzyj\nXbt2KS0tTStXrpQknTx5UqtWrdL//vc/DRkyRN///veVkpIStZoIZP2Uw+HQfffdp5EjRyoYDMrn\n82nMmDHKysoyXRq6aMuWLRo+fLiCwaDpUtAN69ev11e+8hUtXLhQbW1tam5uNl0SLlFDQ4O2bt2q\nVatWKTExUb/85S9VWVmpKVOmmC4NX2DKlCmaNm2aSktLI23l5eW6/vrrNWPGDJWXl6u8vFyzZs2K\nWk1MWfZTgwcP1siRIyVJLpdLw4cPV0NDg+Gq0FX19fXatWuXCgoKTJeCbjh16pTee+89ff3rX5ck\nOZ1OXXHFFYarQldYlqWWlha1t7erpaVFgwcPNl0SLsF111133uhXVVWVJk+eLEmaPHmyqqqqoloT\nI2RQbW2tDh8+rKuuusp0KeiiZ599VrNmzWJ0LE7V1tZq4MCBeuaZZ3TkyBGNHDlSs2fPVnJysunS\ncAnS09N1++23a968eUpMTNQNN9ygG264wXRZ6Kbjx49HAvWgQYN0/PjxqL4/I2T93OnTp7Vy5UrN\nnj1bAwYMMF0OumDnzp1KS0uLjHQi/rS3t+vw4cO65ZZb9PjjjyspKUnl5eWmy8IlOnnypKqqqlRa\nWqo1a9bo9OnTev31102XhR5gs9lks9mi+p4Esn6sra1NK1eu1KRJk3TjjTeaLgdd9O9//1s7duzQ\nggUL9NRTT2nv3r16+umnTZeFLnC73XK73Ro1apQkafz48Tp8+LDhqnCp9uzZo4yMDA0cOFBOp1M3\n3nij3n//fdNloZvS0tLU2NgoSWpsbNTAgQOj+v5MWfZToVBIq1ev1vDhw3XbbbeZLgfdcM899+ie\ne+6RJL377rt64YUX9PDDDxuuCl0xaNAgud1u1dTUyOv1as+ePXywJo54PB4dOHBAzc3NSkxM1J49\ne5STk2O6LHRTbm6utm3bphkzZmjbtm0aN25cVN+fB8P2U/v379eyZct05ZVXRoZl7777bo0dO9Zw\nZeiOcCDjsRfx58MPP9Tq1avV1tamjIwMzZ8/P6oftcfl2bRpkyorK+VwOJSdna0HH3xQCQkJpsvC\nF3jqqae0b98+NTU1KS0tTcXFxRo3bpxWrVqluro6I4+9IJABAAAYxhoyAAAAwwhkAAAAhhHIAAAA\nDCOQAQAAGEYgAwAAMIxABqDPKi4u1n//+1/TZZxn06ZNPMQXQAc8GBZAVCxYsEDHjh2T3f7Z74FT\npkzR3LlzDVYFALGBQAYgahYvXqwxY8aYLqNPaW9vl8PhMF0GgMtEIANg3GuvvaZXXnlF2dnZev31\n1zV48GDNnTtX119/vSSpoaFBa9eu1f79+5WSkqI77rhDhYWFkiTLslReXq5XX31Vx48f17Bhw7Ro\n0SJ5PB5J0r/+9S/9/Oc/14kTJ3TTTTdp7ty5F9w0eNOmTfr444+VmJio7du3y+PxaMGCBZGtcIqL\ni/X0009r6NChkqTS0lK53W7dddddevfdd/XrX/9a3/zmN/XCCy/Ibrfru9/9rpxOpzZs2KATJ07o\n9ttvV1FRUeT9WltbtWrVKv3jH//QsGHDNG/ePGVnZ0e+33Xr1um9995TcnKybr31Vk2fPj1S50cf\nfaSEhATt3LlT999/vwoKCnqnYwBEDWvIAMSEAwcOKDMzU2VlZSouLtaTTz6pkydPSpJ+9atfye12\na82aNVq4cKGef/557d27V5L04osv6q233tKSJUu0YcMGzZs3T0lJSZH77tq1S7/4xS/05JNP6u23\n39Y///nPTmvYuXOn8vPz9eyzzyo3N1fr1q275PqPHTum1tZWrV69WsXFxVqzZo3eeOMN+f1+/eQn\nP9HmzZtVW1sbuX7Hjh2aMGGC1q1bp4kTJ+qJJ55QW1ubLMvSihUrlJ2drTVr1mjZsmXasmWLdu/e\n3eFrx48fr/Xr12vSpEmXXCOA2EUgAxA1TzzxhGbPnh35U1FRETmXlpamW2+9VU6nU/n5+fJ6vdq1\na5fq6uq0f/9+3XvvvUpMTFR2drYKCgq0bds2SdIrr7yiu+66S16vVzabTdnZ2UpNTY3cd8aMGbri\niivk8Xg0evRoffjhh53Wd80112js2LGy2+26+eabL3rt5zkcDhUVFcnpdGrixIlqamrS9OnT5XK5\nNGLECGVlZXW438iRIzV+/Hg5nU7ddtttam1t1YEDB3To0CGdOHFCM2fOlNPpVGZmpgoKClRZWRn5\n2quvvlp5eXmy2+1KTEy85BoBxC6mLAFEzaJFizpdQ5aent5hKnHIkCFqaGhQY2OjUlJS5HK5Iuc8\nHo8OHTokSaqvr1dmZman7zlo0KDIcVJSkk6fPt3ptWlpaZHjxMREtba2XvIardTU1MgHFsIh6fP3\nO/e93W535Nhut8vtdquxsVGS1NjYqNmzZ0fOW5ala6+99oJfC6BvIJABiAkNDQ0KhUKRUFZXV6fc\n3FwNHjxYJ0+eVDAYjISyuro6paenSzoTTj755BNdeeWVvVpfUlKSmpubI6+PHTt2WcGovr4+cmxZ\nlurr6zV48GA5HA5lZGTwWAygn2HKEkBMOH78uLZu3aq2tja9/fbbqq6u1le/+lV5PB596Utf0h/+\n8Ae1tLToyJEjevXVVyNrpwoKCrRx40YdPXpUoVBIR44cUVNTU4/Xl52drTfffFOWZWn37t3at2/f\nZd3vgw8+0DvvvKP29nZt2bJFCQkJGjVqlK666iq5XC6Vl5erpaVFlmXpP//5jw4ePNhD3wmAWMQI\nGYCoWbFiRYfnkI0ZM0aLFi2SJI0aNUpHjx7V3LlzNWjQIP3gBz+IrAV75JFHtHbtWj3wwANKSUnR\nt7/97cjUZ3j91c9+9jM1NTVp+PDhevTRR3u89tmzZ6u0tFQvv/yyxo0bp3Hjxl3W/XJzc1VZWanS\n0lINHTpUCxculNN55p/kxYsX67nnntOCBQvU1tYmr9erO++8sye+DQAxyhYKhUKmiwDQv4Ufe/HT\nn/7UdCkAYARTlgAAAIYRyAAAAAxjyhIAAMAwRsgAAAAMI5ABAAAYRiADAAAwjEAGAABgGIEMAADA\nMAIZAACAYf8Pdy2pK4zdLhcAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hye2f3rMO66g",
        "colab_type": "code",
        "outputId": "7c490686-41eb-4fc2-832a-de3c6f7ef38e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "optimal_epoch(reg_history)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Minimum validation loss reached in epoch 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CM80lTbuPCDh",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "We can see that it starts overfitting in the second epoch as the first model. Also, the validation loss increases slower afterwards."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jQsDKwgZSjqc",
        "colab_type": "text"
      },
      "source": [
        "At first sight the reduced model seems to be the best model for generalisation. But let's check that on the test set using the `test_model` helper function. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IL1F15JyTJjd",
        "colab_type": "code",
        "outputId": "571861cb-daca-41ad-903a-94a2f4b81057",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "# Optimal epochs\n",
        "base_min = 1 # first model\n",
        "reg_min = 2 # model with regularisation\n",
        "\n",
        "# Training on the full train data and evaluation on test data on first model\n",
        "base_results = test_model(model, X_train, Y_train, X_test, Y_test, base_min)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "9565/9565 [==============================] - 10s 1ms/step\n",
            "\n",
            "Test accuracy: 74.81%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2KiX8bRdWHPX",
        "colab_type": "code",
        "outputId": "a0758a4e-0797-4cec-c60a-7e5b0284c51b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "# Training on second model with regularisation and evaluation on data on second model\n",
        "reg_results = test_model(reg_model, X_train, Y_train, X_test, Y_test, reg_min)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "9565/9565 [==============================] - 10s 1ms/step\n",
            "\n",
            "Test accuracy: 74.47%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IynxgnI-Z6k-",
        "colab_type": "text"
      },
      "source": [
        "As shown above, applying regularisation helped with the overfitting but it didn't do much to the model's accuracy on the test data. We'll do a more comprehensive evaluation and compare both versions of the LSTM model with the machine learning models we trained previously, the Naive Bayes Classifier and Support Vector Machine Classifier. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S5ZsWITKp2JH",
        "colab_type": "text"
      },
      "source": [
        "Let's save our models and predictions:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RKy2Ljwop1lh",
        "colab_type": "code",
        "outputId": "26b6fa03-c5d5-4b99-8f54-95dd21cd0fa9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# save first model and architecture\n",
        "model.save('/content/drive/My Drive/Twitter_Project/LSTM_model.h5')\n",
        "print('Saved LSTM model to drive')\n",
        "\n",
        "# save second model and architecture\n",
        "reg_model.save('/content/drive/My Drive/Twitter_Project/LSTM_regmodel.h5')\n",
        "print('Saved LSTM model to drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saved LSTM model to drive\n",
            "Saved LSTM model to drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "soTyYMTNpcP7",
        "colab_type": "code",
        "outputId": "93f87fc3-3a46-4b2d-992b-f180996da896",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# saving predictions\n",
        "X_train, X_test, Y_train, Y_test\n",
        "\n",
        "#-- LSTM model ----#\n",
        "y_preds_LSTM = model.predict(X_test)\n",
        "\n",
        "# Save predictions for evaluation as numpy arrays\n",
        "np.save('/content/drive/My Drive/Twitter_Project/y_predsLSTM.npy', y_preds_LSTM)\n",
        "print('LSTM model predictions saved to drive')\n",
        "\n",
        "#-- LSTM with regularisation model ----#\n",
        "y_preds_LSTMreg = reg_model.predict(X_test)\n",
        "\n",
        "# Save predictions for evaluation as numpy arrays\n",
        "np.save('/content/drive/My Drive/Twitter_Project/y_predsLSTMreg.npy', y_preds_LSTMreg)\n",
        "print('LSTM reg model predictions saved to drive')\n",
        "\n",
        "# Save test data\n",
        "np.save('/content/drive/My Drive/Twitter_Project/y_testLSTM.npy', Y_test)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "LSTM model predictions saved to drive\n",
            "LSTM reg model predictions saved to drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kzci3gytn-h3",
        "colab_type": "code",
        "outputId": "60fa74c0-23b6-4850-f32f-406020a65a10",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Save test data\n",
        "np.save('/content/drive/My Drive/Twitter_Project/y_testLSTM.npy', Y_test)\n",
        "print('Test data saved')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test data saved\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}