Phase_03_report_plus.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final Presentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img style=\"float: right;\" src=\"https://raw.githubusercontent.com/Mt-Kunlun/MarkdownPhotos/master/Res/pipeline.png\" width=\"100%\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "E:\\Anaconda3\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from string import punctuation\n",
    "from gensim import corpora\n",
    "from gensim.models import Doc2Vec\n",
    "from gensim.models.doc2vec import TaggedDocument\n",
    "from nltk.corpus import reuters\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import classification_report\n",
    "from xgboost import XGBClassifier\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "from sklearn import cross_validation, metrics\n",
    "from sklearn.grid_search import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>title_vectors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8476</td>\n",
       "      <td>You Can Smell Hillary’s Fear</td>\n",
       "      <td>Daniel Greenfield, a Shillman Journalism Fello...</td>\n",
       "      <td>FAKE</td>\n",
       "      <td>[ 1.1533764e-02  4.2144405e-03  1.9692603e-02 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10294</td>\n",
       "      <td>Watch The Exact Moment Paul Ryan Committed Pol...</td>\n",
       "      <td>Google Pinterest Digg Linkedin Reddit Stumbleu...</td>\n",
       "      <td>FAKE</td>\n",
       "      <td>[ 0.11267698  0.02518966 -0.00212591  0.021095...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3608</td>\n",
       "      <td>Kerry to go to Paris in gesture of sympathy</td>\n",
       "      <td>U.S. Secretary of State John F. Kerry said Mon...</td>\n",
       "      <td>REAL</td>\n",
       "      <td>[ 0.04253004  0.04300297  0.01848392  0.048672...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10142</td>\n",
       "      <td>Bernie supporters on Twitter erupt in anger ag...</td>\n",
       "      <td>— Kaydee King (@KaydeeKing) November 9, 2016 T...</td>\n",
       "      <td>FAKE</td>\n",
       "      <td>[ 0.10801624  0.11583211  0.02874823  0.061732...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>875</td>\n",
       "      <td>The Battle of New York: Why This Primary Matters</td>\n",
       "      <td>It's primary day in New York and front-runners...</td>\n",
       "      <td>REAL</td>\n",
       "      <td>[ 1.69016439e-02  7.13498285e-03 -7.81233795e-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                              title  \\\n",
       "0        8476                       You Can Smell Hillary’s Fear   \n",
       "1       10294  Watch The Exact Moment Paul Ryan Committed Pol...   \n",
       "2        3608        Kerry to go to Paris in gesture of sympathy   \n",
       "3       10142  Bernie supporters on Twitter erupt in anger ag...   \n",
       "4         875   The Battle of New York: Why This Primary Matters   \n",
       "\n",
       "                                                text label  \\\n",
       "0  Daniel Greenfield, a Shillman Journalism Fello...  FAKE   \n",
       "1  Google Pinterest Digg Linkedin Reddit Stumbleu...  FAKE   \n",
       "2  U.S. Secretary of State John F. Kerry said Mon...  REAL   \n",
       "3  — Kaydee King (@KaydeeKing) November 9, 2016 T...  FAKE   \n",
       "4  It's primary day in New York and front-runners...  REAL   \n",
       "\n",
       "                                       title_vectors  \n",
       "0  [ 1.1533764e-02  4.2144405e-03  1.9692603e-02 ...  \n",
       "1  [ 0.11267698  0.02518966 -0.00212591  0.021095...  \n",
       "2  [ 0.04253004  0.04300297  0.01848392  0.048672...  \n",
       "3  [ 0.10801624  0.11583211  0.02874823  0.061732...  \n",
       "4  [ 1.69016439e-02  7.13498285e-03 -7.81233795e-...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('C:\\\\Users\\\\51645\\\\Desktop\\\\fake_news\\\\fake_or_real_news.csv')\n",
    "df1 = pd.read_csv('C:\\\\Users\\\\51645\\\\Desktop\\\\fake_news\\\\fake.csv')\n",
    "#fake news dataset from https://www.kaggle.com/burakhmmtgl/exploratory-analysis/data\n",
    "df2 = pd.read_csv('C:\\\\Users\\\\51645\\\\Desktop\\\\fake_news\\\\Real.csv')\n",
    "#real news dataset from https://www.kaggle.com/snapcrack/all-the-news/data\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_text(tokenized_list, sw):\n",
    "    new_list = []\n",
    "    nodes = [',', '*', '\"', '.', '\\'', '“', '”', '’', '‘', '，']\n",
    "    for doc in tokenized_list:\n",
    "        new_list.append([token.lower() for token in doc if token.lower() not in sw and token.lower() not in nodes])\n",
    "    return new_list\n",
    "\n",
    "texts = df.text\n",
    "\n",
    "#create mapping for string translate method\n",
    "mapping_table = {ord(char): u' ' for char in punctuation}\n",
    "\n",
    "tokenized = [nltk.word_tokenize(review.translate(mapping_table)) for review in texts]\n",
    "\n",
    "# Remove punctuations and stopwords, and lower-case text\n",
    "sw = stopwords.words('english')\n",
    "cleaned = clean_text(tokenized, sw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get TF-IDF for each text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer()\n",
    "Tf_fit = []\n",
    "# for each text, turn the text into list of words\n",
    "for article in cleaned:\n",
    "    str = ''\n",
    "    for i in article:\n",
    "        str += i\n",
    "        str += ' '\n",
    "    Tf_fit.append(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      " [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      " [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      " ..., \n",
      " [ 0.          0.01034503  0.         ...,  0.          0.          0.        ]\n",
      " [ 0.          0.01270221  0.         ...,  0.          0.          0.        ]\n",
      " [ 0.          0.          0.         ...,  0.          0.          0.        ]]\n",
      "67558\n"
     ]
    }
   ],
   "source": [
    "vectorizer.fit(Tf_fit)\n",
    "tfidf_matrix = vectorizer.transform(Tf_fit).toarray()\n",
    "print(tfidf_matrix)\n",
    "print(len(tfidf_matrix[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PCA Decomposition for the TF-IDF matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\npca.fit(tfidf_matrix)\\nX_tf = pca.transform(tfidf_matrix)\\nX_tf.tofile(\"tfidf_matrix_500.bin\")\\n\\n'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "\n",
    "pca = PCA(n_components = 500)\n",
    "pca.fit(tfidf_matrix)\n",
    "X_tf = pca.transform(tfidf_matrix)\n",
    "X_tf.tofile(\"tfidf_matrix_500.bin\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data for the classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.08367505,  0.41453202, -0.1365418 , ..., -0.00349437,\n",
       "         0.00598387,  0.00284075],\n",
       "       [ 0.10516965, -0.1018543 , -0.07499441, ..., -0.01728784,\n",
       "        -0.01019226, -0.01649624],\n",
       "       [-0.08337797, -0.02132731,  0.09598185, ...,  0.00864918,\n",
       "         0.01190413,  0.03372494],\n",
       "       ..., \n",
       "       [ 0.09876725, -0.04909523,  0.00227706, ..., -0.00569073,\n",
       "        -0.00123435, -0.0120559 ],\n",
       "       [-0.07174388, -0.02140137,  0.10833566, ..., -0.010136  ,\n",
       "        -0.02771981, -0.01208612],\n",
       "       [ 0.31185574, -0.21482338, -0.04275505, ..., -0.00265663,\n",
       "        -0.00849138, -0.00640356]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tf500 = np.fromfile(\"C:\\\\Users\\\\51645\\\\Desktop\\\\fake_news\\\\tfidf_matrix_6335_500.bin\")\n",
    "X_tf500.shape = 6335, 500\n",
    "X_tf500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_ = np.fromfile('C:\\\\Users\\\\51645\\\\Desktop\\\\fake_news\\\\doc2vec_title300_text300_kunlun.bin')\n",
    "X_.shape = 6335, 600\n",
    "X_ = np.hstack((X_, X_tf500))\n",
    "y_ = np.fromfile('C:\\\\Users\\\\51645\\\\Desktop\\\\fake_news\\\\test_y_33.bin', dtype = int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed = 42\n",
    "test_size = 0.33\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_, y_, test_size = test_size, random_state = seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best XGBoost Result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notice:\n",
    "For convience of looking, we omit the hyper tunning process here, which take lots of places."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input: 300 doc2vec for text, 300 word2vec for title, 500 TF-IDF vector\n",
    "### Result: F1: 0.947"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.06, max_delta_step=0,\n",
       "       max_depth=3, min_child_weight=1, missing=None, n_estimators=1400,\n",
       "       n_jobs=2, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "XGmodel = XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
    "           colsample_bytree=1, gamma=0, learning_rate=0.06, max_delta_step=0,\n",
    "           max_depth=3, min_child_weight=1, missing=None, n_estimators=1400,\n",
    "           n_jobs=2, nthread=None, objective='binary:logistic', random_state=0,\n",
    "           reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
    "           silent=True, subsample=1)\n",
    "\n",
    "XGmodel.fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8XVW5//HPc5K26Ugn2lJaCkgZ\nyiDzRQRBuYCgSOE6VEVQposVAQe8IIqtAoJcLl4Bf8gswkWBIoKggAgyKINIoYXSMnekA23plDZp\n8vz+WPukJyd7n31Okp2p3/frlVeStfc+e+2cNitrPWs9y9wdERGR1sp1dgVERKR7U0MiIiJtooZE\nRETaRA2JiIi0iRoSERFpEzUkIiLSJmpIRDJkZtea2Q87ux4iWTKtI5GuyMzeAUYCDQXFO7r7wja8\n5qHAbe4+pm21657M7BZgvrv/oLPrIj2LeiTSlR3j7gMKPlrdiLQHM6vuzPu3hZlVdXYdpOdSQyLd\njpkdYGZ/N7OVZvZS1NPIH/uamc0ys9Vm9paZ/WdU3h/4EzDazNZEH6PN7BYzu6jg+kPNbH7B9++Y\n2X+Z2cvAWjOrjq6bZmZLzextMzurRF2bXj//2mb2PTNbYmaLzGyimR1tZnPMbLmZfb/g2ilmdreZ\n/S56nn+Z2YcLju9iZo9HP4dXzOwzRff9f2b2oJmtBU4Bvgx8L3r2+6PzzjOzN6PXf9XMjit4ja+a\n2VNm9t9mtiJ61qMKjg81s5vNbGF0/N6CY582s+lR3f5uZnuU/QZLt6OGRLoVM9saeAC4CBgKfBeY\nZmZbRqcsAT4NDAK+BlxpZnu7+1rgKGBhK3o4XwQ+BQwGGoH7gZeArYHDgHPM7MgyX2sUUBNdeyFw\nPXACsA9wMHChmW1fcP6xwF3Rs/4fcK+Z9TKzXlE9HgZGAN8EbjeznQqu/RJwMTAQuBW4HfhZ9OzH\nROe8Gd13C2AqcJuZbVXwGv8GzAaGAz8DbjQzi479BugH7BrV4UoAM9sbuAn4T2AY8CvgPjPrU+bP\nSLoZNSTSld0b/UW7suCv3ROAB939QXdvdPdHgH8CRwO4+wPu/qYHfyP8oj24jfX4hbvPc/daYD9g\nS3f/sbvXuftbhMZgUpmvVQ9c7O71wG8Jv6D/191Xu/srwCtA4V/vL7j73dH5/0NohA6IPgYAl0b1\n+CvwR0Kjl/cHd386+jmtj6uMu9/l7gujc34HvA7sX3DKu+5+vbs3AL8GtgJGRo3NUcAZ7r7C3euj\nnzfAacCv3P1Zd29w918DG6I6Sw/Ubcd8ZbMw0d3/UlQ2DvicmR1TUNYLeAwgGnr5EbAj4Q+lfsCM\nNtZjXtH9R5vZyoKyKuDJMl/r/eiXMkBt9HlxwfFaQgPR4t7u3hgNu43OH3P3xoJz3yX0dOLqHcvM\nTgS+DWwbFQ0gNG557xXcf13UGRlA6CEtd/cVMS87DjjJzL5ZUNa7oN7Sw6ghke5mHvAbdz+t+EA0\ndDINOJHw13h91JPJD8XETVFcS2hs8kbFnFN43TzgbXcf35rKt8LY/BdmlgPGAPkhubFmlitoTLYB\n5hRcW/y8zb43s3GE3tRhwD/cvcHMprPp51XKPGComQ1295Uxxy5294vLeB3pATS0Jd3NbcAxZnak\nmVWZWU0UxB5D+Ku3D7AU2Bj1To4ouHYxMMzMtigomw4cHQWORwHnpNz/OWBVFIDvG9VhNzPbr92e\nsLl9zOz4aMbYOYQhomeAZwmN4PeimMmhwDGE4bIki4HC+Et/QuOyFMJEBWC3cirl7osIkxd+aWZD\nojp8LDp8PXCGmf2bBf3N7FNmNrDMZ5ZuRg2JdCvuPo8QgP4+4RfgPOBcIOfuq4GzgDuBFYRg830F\n174G3AG8FcVdRhMCxi8B7xDiKb9LuX8D4Rf2nsDbwDLgBkKwOgt/AL5AeJ6vAMdH8Yg64DOEOMUy\n4JfAidEzJrkRmJCPObn7q8AVwD8IjczuwNMV1O0rhJjPa4RJDucAuPs/CXGSq6N6vwF8tYLXlW5G\nCxJFuigzmwLs4O4ndHZdREpRj0REpBsws7Fm9piFdVKvmNnZUfmHzewfZjbDzO43s0FR+bDo/DVm\ndnWWdeuUhsTMzop+GLdH49vTox/M3wrO+aSZzTazN8zsvILy26PymWZ2UzSfXgSA6N/EkmiB3JLo\n38nnon9fjdHiu0Yz29fMtjWzWjN7Nyrz6HOjme1pZs+Z2YbonIVRcJpozP91M1sfHUuLq4i0h43A\nd9x9F8JU6m+Y2QTC0Op57r478HvCUC/AeuCHhLVW2XL3Dv8gjKt69HlV9LWH6jiE6YMbC8rrgQnR\nsbmERWH5Y1d3xjPoo+m9/BawKPpHu5IQY9iJEHOoi96jrcp4nbOBmYR1FOdEZXsSAsvTCWtF9k95\njZvYNCb/FrB39O/l9ejfzCvAfwCPA/sSprzOBtYA9xBiKhuAt6LXOw/oE339DPBM9PXvgNejrycQ\ngt5Vnf1e6GPz+iDEzw6PfofmwxRjgVeLzvtq1r8nO7xHYmbXsmnacRVh1W1ePmBzRHQsrxq4Kvp6\nNM2nJ349g2pulsxssIWUHK9Ff7nPiXp/a82swcwWm9nGgr/aHbiE8Mv3AcJ7+TngBeBawlqONYTU\nHKXuuxshOLs/8GHg02Y2nrCSeqq770lYBf6zlEe4hTCVtQFYByyP6vZp4AlCY/du0TWjCLOP/k5o\ndKoIAXnc/VJ33xCd9zghiSSExuOR6JxXo3t8JaVuIu3GzLYF9iLM3ptJmHgB4f/f2PirMqxP1GJ1\n3A3N1hOmaMZZ7e6DzOxBwmyUQusJ8/0biy9y95Lz3gcPHuw77LBDa6rbJa1du5b+/fuXde6MGTPI\n5XKYGWbG+PHjeeWVV9i4cWPs+dXV1QwYMICamhree+89BgwYwMiRI3n33XfZeuutWbx4MRMmTMDd\nefHFFwHI5XK4O/3796exsZH169ez5557Yma89NJL1NTUsNNOO8XeD2DFihW8//775N+jRYsWYWas\nXr2aYcOGMXToUJYvX87KlSvZfvvtE18HYMOGDcyZM4dcLscOO+zAG2+8wa677srs2bNpaGhg3Lhx\nzJ8/nzFjxrBu3Trmzp1LdXU1uVyOESNGMH/+fCZMmEDfvn2bve5LL73E4MGDGTduHG+//TarV69m\nt912Y+3atcyZM4cRI0Ywduym/7+VvEfdQU97Hui+z9TQ0MCcOXMYNWoUQ4YMYf369cydO5e6ujqG\nDh3KkiVL2HPPPZvOX7ZsGevWrWObbbap+F4vvPDCMnffMu28rtaQQBiSyK8JKOTALoSphs0PxDQk\nZvYWsB3A0KFDuemmm1pb5S7vtNNOo2/fvixYsICqqioaGhpoaGhodk7v3r0ZMmQIy5Yto6GhgVwu\nR69evejfvz/77LMP22yzDbfeeiuXXHIJf/nLX3jxxRdxd0aOHMnOO+/MzJkzOfroo3n++ef5zne+\nw913382dd97J2LFjefPNNwHo1asX++67L2+//TbXXnstAKeccgo1NTVcc801ifWfN28eP/3pT7n0\n0kvp06cPF154IR/60Ic46qijmDp1alP3+dJLL2XEiBElfxaLFy9m6tSpVFdXc8EFF3DxxRfzi1/8\nggsuuIDa2lomT57MzTffzJe+9CVuueUWzj33XB599FHWrl3LX//6V9atW8e9997b7DV/9KMfMXfu\nXG688UZyuRx1dXX813/9F/Pnz29qPA8//HC+8hV1SiRbGzdu5KKLLmKvvfbi2GOPbXF8wYIF/Pzn\nP+fyyy9vKnv00Ud58803Of300yu+38SJE19w933TzuuKK9tnEhqSOOXmM2qhg9vLzDU0NPDd736H\nd999l8I/BoobEIATTzyR4447nvPPP4/Fi0M2jp///H+56KKfUFdXz8SJx7Fhw3qqq6v57W9/x8yZ\nMxgwYABmxpe//GWmTJlCVVUVt9xyCz/96aVA+MfZt29famtrm3o6r7/+Os888ww1NTUVPcvYsWM5\n7rjjmDJlCjU1NWy77bZUVVXx5z//mZNPPpkDDzyQp556iquvvpof//jHbfipbfLQQw9x7LHHNjVM\nw4cPp6qqilyu+WjvNddcw+zZs7nuuuuajvXu3Zsrr7yy6ZxJkyaV7HGJtAd35+qrr2bMmDHNGpGV\nK1cyePBgGhsbueuuuzjyyHLzh7afzmhI1lG6R/JpklM0tPwtmazZKlorJ+lDFxUaje8ybNgwzj//\nfE444QRqa2ubndOrVy/q6+sxsxD8yuWorq6mrq6Ou+66i6eeeor58+dj0Q/ie987lw0bNlBVVcXl\nl/+MESNGUFtby1ZbjWLevLmMHj2aOXPm8Je//IVcLscvf/lLZs6cyTXXXM15553HokWL2GabbRg0\naBCLFi2irq6OM888k3vuuYcFCxZQX19Pr169aGhoYMiQIanPePjhh3P44YcD8Jvf/IZhw4Zx2223\nceqppwLw0Y9+tGSvplJz587llVde4ZZbbmHdunW4O7W1tVRXb/ovceedd/LYY49x5ZVXMmjQoKby\nVatW0djYyODBg5k2bRq5XI79998/7jYi7WbWrFk8/vjjjBs3jnPOCRMFTzjhBBYtWsSf/vQnAA44\n4AAOO+ywpmtOO+00amtr2bhxI88++yxTpkxpNgTbXjqjIRmUcnxJdE6/mGN3Az8p8z6raZ58rlta\nv349kyaFjtjy5cub/mEUy/dKBg0axAcffIC7U1dXB0BtbS3z589v+j5/vrtTU1PDfvvtx4oVK8jl\ncjz55JPssMMOHH/88UydOpUnn3ySXXbZhcGDBzf9Mr/iiivI5XKcfvrp/OIXv8DMOPjgg5kxYwb9\n+oW37YEHHmDixImsX7+evffeO/U5839VLV26lGeeeYbLLruMBx98kJkzZ7L77rvz8ssvs9VWW6W+\nzg033MB7772Hu3PGGWdgZnzlK19h7dq1AJx77rlNP6chQ4bQ0NBAfX09q1atAsLQwfHHH88hhxzC\nE088gbvz7W9/G4CRI0dy9dVXM2/ePC688ELMjJqaGn74Q+2kK9mbMGFCi2HXvGOOOSa2/Prrr8+y\nSk06oyFZBZT6EzUp1UQjISVFj1VXV8f555/PvHnzmn7Jr1mzpun4Bx980Oz8fC8kr3///k3nVFVV\ncfLJJ3PdddcBoeEYPXo0CxeGfH8HHXQQjz76KLW1tbz11lscf/zxPProo6xatYq5c+dy8cUXU1dX\nh5kxenRI2vryyy+zxRZbMH36dD760Y+y22678fGPf5xp06Zx22230bdv36Zg+B//+EduvfVWGhsb\nuf/++1m4cCFnnnlm4rNfdtllrF69murqak4//XQGDBjA5MmTueGGG2hsbKRXr15Mnjy55M/viiuu\n4I033sDMGDJkCJMmTWLgwIFcf/311NbW0r9/f7bbbjumTJnS7Lo77riDvn37MnHixGblZ599dux9\ndt11V6ZNm1ayLiKbk64YbH+CMMe/uEfihKm/i4ov6O7B9jVr1nDyySc36zHkh6ji5HI5GhsbmwLr\nAH369GHDhg1N1+6xxx6sWbOGd955h8bGxqYhrcbGMOmtX79+rFu3jqqqKo444gjWrl3LU089xciR\nI+nduzcjR47kjTfeoLa2lpEjw6zXXr16sWzZMlasWMGYMWOorq5mp5124qSTTuKcc85hxYoVmBlf\n+tKXWvxSFpHupzsH24cRktDFzVVLH2zv4pYvX84pp5zS1EhUV1czevToZo0IwBZbbMHKlcXZuYN8\nY1AYWO/duzd1dXVNQ1bLli1jzZo1TedUVVUxduxYNmzYwKJFi5qGxxobG3nuued4//33gTDrqXfv\n3uy8885UVVXxq1/9qll8IEm+5yMim5+u2JA8T1jZHteQtDrY3hVmba1Zs4ZTTz21WU9j48aNsTOt\nkhqRJL179272ugsWLGhxzgcffNA0o6pPnz6MGDGCU089lT320HbaItJ6ndGQpN1zJGG9SJzY+ImZ\nmbccB2oWbO/MWVuzZ8/mxz/+cVPAFzYNT0H8L/1SBgwY0BQ7ycdJli9f3rTosLGxkT59+lBfX4+7\nM3DgQLbffvsWsQERkfbQGQ3JRpqnPyn2BM33jM5zNqWoKDacaHOeruiFF15o1ojApuGpYqeffnrJ\nYaLddtuNmTNnNn1fVVVFTU0Na9euxd3p3bs3Z555JgcddFD7VF5EJEVXHNqaRNi5rZjTMm1KXqmG\nqVM98sgj3HnnnU3f9+/fv0WjApt6FjfeeCMAAwcOZPXq1S3Oy68iz9uwYQPHHXccX/jCF9q55iIi\n5emK+5EkTf/NAVPiDrj7e5nVpg3uueeeZovoevXqFduIAE3TePPxkrhGBGi2hmTIkCEcdthhakRE\npFNl1iMxs7MImXlfJUzb3Ru4oIxL55OcIuXkhHvFxUia6ehg+5w5s7ntttualRWu+UiTy+UYP348\ns2fPbiqrqqqiT58+nH/++ey+++7tVlcRkbbIcmhrMmEoai1hFla5CwteIqz/2Drm2APApTHlcTGS\nTk2R8sILLyTGQcoxfPhwLrvsMiZNmsRvf/vbdqyZiEj7yqQhifYc2QGYQ5iyuxYYSlidnvbbNSm1\niRM2HoqzrILXydTSpUu55JJLePvtTYvwCxcOxsmvASmUXwSoRkREurpMYiTufgbhF/984CDgi4Rd\n7t4sdV3kcMLMrmJGQsOXNqzVkZ5++ulmjQjEZ+QtVLyiffTo0fzkJ+WmFBMR6VxZ9kiqCUNa9wGX\nAx8Q0p68T/I0Xgi5uP4FHFxU3giMb/fKtpOlS5dy+eWXM2fOnFa/xt13390s+6yISHeQZY+kkTCs\n9QLweeBjhBQnw1Iu3xr4SEy5EfbY7pKefvrpZo2ImdGnT6mUYs0deeSRakREpFvK8jdXjhAAP5cQ\n+L4b6FvyimAjyfuRvA3sWVxoZoPdvTinSIelSHnuuWe55ZZbCuuDuzclUUxz3nnnccABB2RUOxGR\nbGXSI4mGtiAE2L8K/J7Q0xhGeuPVi/iAfI74oDrED3k1W4hhls3HsmVLW+T8rzRkM358lx2xExFJ\nleXQVgNhk6r/dvcxwFPAG8QH0gutA+L+lHfguYRr3mplVdvs6aefZtmypPYtXa9evRg6dGg71khE\npGNlObRlhKD67WZ2KNC7zOsGEz8EZsAhcRe4+/utqWBbffGLX2xaaT5mzBjWr19fUaMyfPhwzjrr\nrKa9QkREuqOsU6TMA+qB2YQexavE9zYK3R+dW8yBd9qzcm1VmK7kqquuYvjw0stWdt9996Y9yL/2\nta9xww03KIW7iHR7WU7/zRGGt54FphKC7RCGrkpNZ1pDaHyK6+aE4bFWae9g++TJX2/2/VNPPcVr\nr71W8prXXnuN994LacF23nnn9q2QiEgnyXq+aSNhSOr3wChCY5CWqXduwjlGwj4lZjYqJnHjwLhz\n20u+Qci74oorUq+pr69nxYoV3HvvvVlVS0Skw2UdbL+TsKr9WUKalBzxw1aFPgK8knAsdtvAhOy/\nmc7aau1i+sMOO6xV14mIdFVZxkgM+B4hrrENoYewnPRcW/2A7ROOlZ8+twuqqqri61//evqJIiLd\nSJYNSY7QmIwDniQE3p30xmAQyTO8ys0gnKmrrrqq4msOPPBApk2blkFtREQ6V5bB9rz7gBMIMRIj\nedV63o5AHS2nABvxqeU73GOPPVbR+bvvvrt6IiLSY3XEgsRbo+IGNgXfS1lP8uys9XGFVsZCDPf2\n+YDk/dbjHHzwwfzkJz9h4MBMY/8iIp0my6GtDwgLEi8jDGtVEVagp/0Wfpv4hYdOWI8SJ24BR4uN\nrdrjo1Lf+ta3Kr9IRKQbybIhGURYM7ILUBt9XU5SqZeIj5E4cFzCNUkbW7W7Soeocrms13yKiHSu\nLJM2VhMak3uBbwI1hMYg7Z77J70s8N2EY71aUc1WKV4/IiKyuct6h8R64FPA09H35Wy1O4T4OIqR\nPGsrLRFku+lCmzGKiHQJWY67zAXeAw5298HAT4HHSG9IagmB+Th/TShP/e3eHoH2Sqf9HnnkkRWd\nLyLSHXXFLfmGkTyz65mE8m0JQfpC7TpNyqyyab/33HOP4iMislnojN90afccQ3LMIymAvmXaue0x\nW6uSab9z584t+1wRke6sIxqSEWY2H/g2Yd/2tHs+XOKcpERVnbaxVZKFCxd2dhVERDpEZg2Ju29L\niIcsAfYFVhFWrKfFMz5NyMnV4iVJyP5L+iLHNjvuuKSZx/HWrl2bUU1ERLqWzGIkZnYWYcrvW4TZ\nW0MIDUtDyn2fJjQ8cZKSObZ+r9syVTpba8SIERnVRESka8lyaGsyIXX8Jwip5G8BHic9aeNuJMdI\nXowr9DJ+y7c1LUqldtttt9ZdKCLSzWS5IHE8sB0haeOuhOnA5USra4jfjtcJ6Vbi7rdVTHG7pUj5\n4he/WEa1NznyyCOpqkrbv0tEpGfIckHiPGA+sIe7X1lwOG0V+gJaZv6FEAc5J+GauKzA7ZYipXBv\n9nIo06+IbE6yXEcykJAi5WYzGw7sB7xJ/H7shcaWOD4poVzLzUVEOknWSRuXAucCZwF/j8rTeiSv\nkLyyPWlObWZzbStdzf6pT30qo5qIiHRNWSdtHA581t2fZ1N8JC3YniM+RgJhy94W3H1Ra+pZjko3\nsZowYUJGNRER6Zo6YmOre8xsIWGPkR2Jj38Ueoz4XssGQi+nBTMbFVPcLNjeEZtY1dTUqCERkc1O\nlkNb84E93f1Vdx8NTCUMcyUNW+WdSnxDsg44L+GaYTFlbU6RUqkzzzyTIUOGVH6hiEg31hm5ttJS\nvj+fcM5goE/CNUvaVKMElc6+2mOPPbKohohIl9YZDcm6lOMHE1+vHHBSwjWZrGyvdBOrQYNiR95E\nRHq0zNPIR/GLfxLiG43AFimXvEP8lrxOQs+jnJXtraFNrERE0mXWkERJG/PG5L8ws/UkD1FB2K+9\nPuYcAx4Ejm9dfSo7v9IYydlnn13ZBSIiPURX3NhqS8IuicUNiZOwBsXMLKZX0iJFSnswsxY9lerq\naj7+8Y+3zw1ERLqZTGMkZnaWmc0ys2lm9g8z20D6gsRXSZ7ZlbR37fCYsnZLkSIiIsmy7pFMBo4i\nZAEeB0wEvkXptSQ7EGZoFTPgxug1irV7sD0pUaPiJiIizWW5H8m1hEZhFjCbMFtrP9IzAL8DDCAs\nXizkwF/jLsgi2F5JosZRo+LWQ4qIbB6y3CHxjOj1DRgB7AFUkd549QLi0sIb8b2RpJXtHebqq6/u\nzNuLiHSqzBqSqEdihISKvyYMc60iPVPv9iQPfX0moXxxTFmrU6SIiEj5sgy2Hxx9Hgd8BTiDsJYk\nbf5UH2BNwrG3EspPiClrdYqU+++/P6WKIiKSl+XQ1q7RlycBexFSySdl9S2uU1ywHeJ7HpCedqVs\n7777LjfffHPicWuvecQiIj1E1kNbEJI1ngH8ntDbSPtN7MTvL+LADQnXzG5NHePcfvvtFWX8FRHZ\n3GUdbM/f48OErXchPUaSA0bGlBvxqVMAXqy4gglmzZoVW15dHeYIFE8Q23PPPdvr1iIi3VJHJG38\nIXAhIYgO6T2S90geqto1oTyu4Wmm3CD7+vXrY6/v169fbPmUKVPSbi0i0qN1xNDWjwhxkrg9Q+Js\nSWhM4iSNOaXO2ip3/5GkJSl9+6btxyUisnnKemirATja3b/n7tXA30gf2qoHRse9JLB1wjWpG1uV\nq1QwvbhX0r9//9bcQkSkR+mQ/UjMbJSZzQc+QvrQFiTn40pKQd9uqz/q6+O3lB88eDA/+tGPyOXC\njyyXyzF16tT2uq2ISLeVda6t+cByd18GjDGzKcD3KJ1rq5HQMBQ3OAbEBypgeRvrCcALL7yQeOyy\nyy4D4J577mmPW4mI9BidsUNimg0kx0KSeiqpwfZyPPPMM+3xMiIim5UO2Y+kaJfEtKh1IyEnV5xK\n6tsiRUqapBlbIiKSLNOGJG6XRDPbSHJDka9TQ8I5sb/p3T1ultdqCvYpKWdBel1dXXyFqrvi/l8i\nIl1DZwxtrUo53kDYITHOznGF1k55S1599dXY8k9+8pPt8fIiIj1SV4yRVBG/p3sjYW+TOHE7JFZs\n7dq1seWTJk1qj5cXEemRumJD8hbxU4RzwCFZ3fThhx9OzLE1YMCArG4rItLtdUZDkjSFN+/DJMdu\nnkwoT91qNy09yooVK9JeQkREYnTFHsnjxE//9YTyJBWlSBERkdbJtCExs7PMbJaZTTOzf5hZqTUi\neYcknGPAwwnXxMVIWpUiRUREKpN1j+QCwiLCPkB/kgPphRYBcUvMHfhQwjU1ra2giIi0TdbZf0cQ\nUsI/C5wC/J0wvbeUVYSFi3FmJJQvaE0dm910VdqsZBERiZNlj2S36PN44DxCY3JwGfd04jewMuDU\nEte0yaBB8W3XXnvt1daXFhHp0bJsSPJxi88CxwG3ACvLuOdIkle+P5dQvm1MWYsUKaU2tEpy7rnn\nplRXRGTzlklDEg1r5XsV1xJ2NpxLeTOv+pE8/DUhoTyuB9Ms2N6aGVs1NTWJOyOKiEiQSUMSbWqV\n36P9M+5+ZQX37EfyOpJ3E8r/VUH1RESkHXXYxlbAt4HBpG9s9Q7xyRmdhC14o/1O2mT48JYziEeO\nbJfs9CIiPVrWDUkjcDtwA6GnUc7Sv22I33ekEbgj7gIzm9jaCubddtttLcrOOOOMtr6siEiP1xEN\nyeeBk4GbgLdJj5HUEZ/9tw7oHXeBu9+bVpFSwfZly5bFpkjZZZdd0l5WRGSzl+VGG8Oi178DGEWY\nRVVOj2Q+8bOw+gBfqOD+LVKkJHnooYdalNXUaI2jiEg5MuuRuPtAYB2hUVgD5HO0p635GEN8A2fA\n/hVUoewUKXFb7CZlAhYRkeayHtrqBywGXiEhUB5jKWE1fDEDfhN3QRTMb7WVK1e2KEvaLVFERJrL\nMkXKK9GX+TQpXwI+ID1FyljiFyQ60Xa9MeIanrKp9yEi0noVNyRmNsTM9ijj1PzeIfOBR9z9+TLv\nWUdocOIkXft+GfVJlMt1xWz6IiLdQ1m/Qc3scTMbZGZDgZeAm83sf0pdEy1KBJgM3GFm8wnJGNPu\nmc8U3KIawJZJVYwpS02RIiKepfgqAAAbMklEQVQibVfun+JbuPsq4HjgZnffB/j3Mq/9wN3fc/cx\nbEqTUkojybPJtksoPyymLDVFSl5DQ9pom4iIJCm3Iak2s60Ia0L+mGF9AFYAy2PKHdg34Zqk9PJl\ncXVPRERardyG5MfAQ8Cb7v68mW0PvF7mtXeY2SNm5sA40teS9AEGxJQ3kLCY0d3LnREWSzESEZHW\nK2tBorvfBdxV8P1bwH+UcelGQhr5DcBfCbm2YlenF1hA/OysKkK8ZWbxATMzb0O3Qg2JiEjrlRts\n39HMHjWzmdH3e5jZD1KuuZYwHPVr4Ah3Hwn8lPj0J4UGEvYtafGSJO+EGLdnezOlgu2rV2t7dxGR\n1ir3T/HrgfOBegB3fxmYlHLNq9Hnd4DPm9kG4EDiEzIW2gj0jSl3klOkxK07GRhTJiIi7azcXFv9\n3P05a56wKm0R4GRgCXAu4Zf6RGA/0huvrUhubCrZZWo1BT2VUrm2RESk9cptSJaZ2YeIpu6a2WeB\nRUknR8NaOxB6CncTfqnvB7xJiJfE9TjyFhO2241rTMYmXKP5uyIinaTchuQbwHXAzma2gJAO/stJ\nJ7v7GWZ2CqGx+SyhR3IFoYFIMyyh3EiOr7R5YysREWmd1IbEzHLAvu7+72bWH8i5e8nodNQjqSY0\nHE8AzwEHUV4a+Q0l6nVgQvlOwGtlvLaIiLSz1GC7uzcCZ0Zfr01rRKLzziAMN70D/BthY6sXCWtP\n0mIrVcQPaznJPY+4WV5FdYqfsbV+fdyuvtCvXyXhGBGRzVe5s7YeMbPvmtlYMxua/yjjtccC9xGG\nwVYTGoP6lOuSZls5ybGVxWmvk5Qe5dVXXy2+DoCLLroopZoiIgLlx0hOjj5/o6DMge1LXNNImLX1\nBeBhQsLFHPFTdQvVRecV160RGJ1wzTBa9laazdpKMmNGfHaV7bcv9WgiIpJX7sr2pGSJpcwnNAhL\n3H2MmU0hxDj2oHTQPWnluwEPAnvFVbEV9RMRkXZQVkNiZifGlbv7rWVcPsLMphNSyPcm5NIq5Q/A\nocAWReVVxKRHicQleSyLMv+KiLRNuUNb+xV8XUNI2/4voFRD8j/AlcCzhOGqfCLGekqvbt8vukex\njcDecRe0Jc9WUrBdRETKU+7Q1jcLvzezLUjYP73AZEJcYyIhVnEToTeSliJlBKHhKWbAM+XUN9Ji\nY6tmLxYF3Pv3j9tDS0REytXatLfrgPFJB6N1JOMJw1G/IQTJLydkAE5L2thI/PCXUdkK9pIbW4mI\nSPsoN0ZyP5sC2jlgAgVp5YtFK9s/GZ27t7svi4LtAKsonSIlqU4GnJVQv1Ft3ZNERERap9wYyX8X\nfL0ReNfd57fynoNSjuen/8bN3poDfKK4UI2IiEjnKXdo62h3/1v08bS7zzezy1p5z7QYSUOJc2Kn\nDZvZqFbWRbO2RETaqNyG5PCYsqPasyIFflHiWOwqwXJ6JJWmSBERkfKUHNoys68TZl9tb2YvFxwa\nCDyd8toDCVvrvmNm/SgvYSPAKYReSVzditeWpN2/SVKAfcmSJRW8pIiIFEuLkfwf8CfCFrnnFZSv\ndve0RYBLCckUvwAMJeyouA1wSMp9ZwF70rLRcNITPhYqK0XK9OnTK3hJEREpVrIhcfcPgA+ALwKY\n2QjCYsEBZjbA3efGXRdN/90p+vZZwiLEPoTG5X3S9yWJm9VVT5jx1W6+//3vx5ab5geLiJStrBiJ\nmR1jZq8TNrT6GyE9/J+Szo/SyEMYonoCmBd934/kjavyRhE/Y6sRWFhOfcs1a9as2PJx48a1521E\nRHq0coPtFwEHAHOiBI6HUSJGYmavRF9WAbuwaRFiL0KPpJQq4pMw1gBPJtxvcMprVuTnP/95e76c\niEiPVm5DUu/u7wM5M8u5+2OEOEaSwl/4hZtOVZG+jmQNydl8v5pQHrfKvkWKlOIZW3FyudYu9hcR\n2TyVuyBxpZkNIDQQt5vZEsoLfDcShrLym35UkZ79N2kjECM5LcuamLJmwXaFPUREslHun9/HEvJr\nnQP8GXgTOKbM188BH6ngnlclnNNIck+lVfu1tyFpsIiIRMpqSNx9LWHb3EPd/dfADcRn6M2ff0bB\nt0uBS6LzG4B3U253FvHJGZ3khI8TUl6zbI2Nje31UiIim4VyZ22dBtwN/Coq2hq4t8x7VANTCTOx\nqoC0KVH9iN+Otw6InW4MxG+8niJumq9iJCIilSn3t+Y3gI8SreNw99cJ+4akaQBOJEzbbaC8LXGT\n0p3UkLAfSTkbW5UbbBcRkcqU25BscPemoSwzqya9USgeniq3IRld4lglIfMWKVK0F4mISPsrtyH5\nm5l9H+hrZocT9iK5P+Wa4jTzGyivIfkn8etFVhNW2bdgZhMTzhcRkYyV25CcRwiazwD+E3gQ+EGp\nC9x9W0IP4lZgOmHab1zso9gEYExMeS/goIR7lRuvKb6uNZeJiEiBtOy/27j7XHdvBK6PPirRCHye\n0BvZj5ANOC79SaGngd1jyvsAKyq8f8U0a0tEpDJpPZKmv/TNbFolLxwlbnTg18AR7j6SkEU4bc/2\nfYhP6mjApZXUoZQ1a+LWMGrWlohIpdJWtheGppNWnMcq2Lf94+6+rOBQ2g6Jg4gPqhsJCR/NbFiU\nwqVQixQpm86H1asVQhERaQ9pDYknfF2ugcDfzWwGYTbWfqT3gqYT8ngVNzhOcoqUuHQtJVOkaAhL\nRKR9pDUkHzazVYTeQN/oa6Lv3d3TEjAOBYYQVsX3JjQi+QbJo4/ihmVv4oPyRnJjlhZ3ERGRjJTs\nHbh7lbsPcveB7l4dfZ3/vmQjEsVIICxGvB74CSHo3nRKwv0XEFbCt6gOcFvC7dKGy8qmnoqISGXK\nzf7bGp+PPm8NnERYmV7Yc3DiYyFjCCvpizmhhxOnki14Aaivr48tHzGinAX7IiKSl9kUJXfP/9Jf\nQFhgODt/KPqctMbcSd6zJGn679L0+jQPuC9dGn/Jddddl/ZSIiJSILOGpGBoq4Gwb/spxGf1jbMh\npixHcpbfgWllxelRnn322TKrIiIipXTEogkH1rj782We30hylt8zE8qPiykrOb938eLFZVZHRERK\nyXJoK78nyVTgDjObT5iNlZY2cT1hplec/0sof6DyGoqISHvoiB7JVOB3hKB7OdYR9iSJ80RCecXP\noTxbIiLtoyMakpHAjoQGAspLP78wptyBLyRcs0Olldq4seKJXiIiEqMjgu3vEXZXzAfQyxnaiuuR\nGPE5uAD+EVPWIkVKYSdEDYmISPvIskdycPR5JKHHsI7y0qwMArZIOHZfXGHCDonNgu3Fs7b69u1b\nRlVERCRNZgsS3X1XM3PgEHd/FsDM3gG2oXSvZECJ46V2TxQRkU7QFXOmO8kNXJ+OrIiIiKTrig3J\nCpIXLu4UV2hm+1Z6k6QUKSIiUpkOaUjMbFS0jmQM6cH2PwLLEo7NjCt093+m1UHBdhGRbGTdkDQQ\nehhbAqMob8/2mcTn2qoHHqng3iVTpCjYLiLSPrLM/gvwbeAPwKvA84S9Rqop3YDtSXyDU0VK2pMi\nzTa2EhGRbGTdkEwGjgLWAuOAicD3KN2QVBE/Tdgof3W8iIh0kKwXJG5PWPvx5ShpY330UcrHSN4h\nccf2qNsjjzzCjBkz2uOlREQ2e1n2SF4lNAjbAj8wsyui8rRFiWNK1GtYe1Rszpw57fEyIiJCtg3J\nZEJK+ImEBmUScChhiKuU9wnB9rg0KZ+r4P4tUqRA84C7iIi0XSYNSTSstSNhOOqPwGJCT2MlIV4S\ntxFV3hCSh9z2qKAazYLtakBERLKRSYwk2oukIfr4OKEn8ffocP+UyxtL1OvtuEIzG9WKaraQy3XF\n9ZkiIl1blj2S/GvfT9hq9yDSFyNC2H89ac3JN4C4AMeSVlSzhS233LI9XkZEZLOSdY9kI3AkcDLw\nIrCc9Flbg0vU65SE8orWi6xeHb8c5dRTT63kZUREhGxXtucIvZLbgC8TYhblpJGfQRjeinN8QvnS\ntBctTI/y+uuvtzhuZuy3337ptRMRkWaynLXVSBjK6gtcAtSUed0+xA9rOfBnYHyLA/H7kSQG9ONi\nIaZovIhIq2TZI5lPaEz+gzDl93pCsDytV5IUbHfggQru32JjKxERaX9Z9kiGRa9/B2GXxBzxa0OK\nVREflDfgB3EXmNlEd7+3lfUUEZE2yKxH4u4DCQH3Y9x9sLsPAp4iBODT6lQbU25AUmPxfiV1a2hI\n2u5EREQq1dH7kXwE6JVyeg7onXDswYTyl1tbNxERaZvMhrbM7CxCL+IFoI6wYh1KLzgEeIewCr44\n4J64Ba+7fxBT3CJFSj5O0tjYclJYfLxeRETSdMVcW0NI7rV8uIL7J6ZI6dWr5ctr1paISOtkMrQV\nrWwfT2iobgcOAD4LbEP66vYVJM/s2i7hflu1rqYiItJWmfRI3P0MM/skoaHa292XmVkOOJCQeyvt\nvkmNzREJ5VsDi1pVWRERaZMsh7YGEtLB32xmw4H9gDfLuG40yT2lDyWUVxTg0KwtEZH2k+WsrUGE\n1CXnAmexKftvmgXAhoRjDyWUL0x70cJY+saNLWcgK9guItI6WcZIqgkLEZ8AziMEyscTv0ak0FbE\n95Qc+GfcBe4eN6yVmCKlT58+cXVOqZaIiMTJMvtvfprva8AxhB6KUXpTKwg9l3Ux5UbYGKvlAbOx\nMcVKkSIi0gGyjJFASBk/HLga2JfQQxlLSOSY5DDiG7h1hPhJnAVtqKOIiLRBlhtb5QjDUcMJa0c+\nTOiNpKVI2Uj8yvbewF4J1yjYLiLSSbLe2MqANUAfNq1UT9prJC+pcasGpicc27bCKoqISDvJemOr\n/OsfzKZeRstId3N1xPcwGgmr5eO02KOkWOGkLKVIERFpP1k2JHXR5y0Jq9VzwHrSh6FyxC9IrALu\nSbhmRkxZs6C+UqSIiGQjy2B7fijrs4SG4Q+EXRLTfmPXkpz996SE8sUxZc1ybYmISDayXkcCcDRw\nQ/S1k94j2SKh3ElekDi03LpdddVVLF2ausW7iIiUKctg+7uEgPs17j4G+CkhRXxaQ7Iouq6YAbsn\nXFOXUN7Co48+Glset4+7iIik67CNrYBvE6YBpw1tjSC+Yagnub47t752QXV11ktqRER6pqx/e64G\n/gjMIezXXrxZVdI1g2PKq4E/ARNijr0VU5a4sVWcMWPGlFE1EREplnWPZBlwJHAycBPwKOnrSJYS\nZnnFeSqhPK6JqChFSk1NTUq1REQkTpY9kmGEVCiz2LSmpIr0GMnWxKdQqSc+BxeEBktERDpBZj0S\ndx9IaDQOJOzB/hHgSdIbkt7E91p6k5zwUdN8RUQ6SWYNScEU4NuAL7v784QGIi3R1XLg5ZhyB76b\ncE2beyRDh5Y9g1hERApkNrQVbbd7KjDR3edUcOlQQmA+znsJ5cNoZWMyaNAgbr311tZcKiIidND0\n3yJpjdeshHMMuCjhmvdjytL2PQGgb99SGe1FRCRNh60jMbP5hDhJ2jqScSTP7BqRUH5ETNnqmDIR\nEWlnWTck84Hl7v5ewer2tP1I8unn4zwQV+juSalTREQkYx2ynDta2f5Pwna7aYsSq0hONf9JkvNt\niYhIJ8i0R+Lu27r7snyPxN0HAdflDxM/FXhQiXr9rr3rWFtb294vKSKyWcm0ITGzu81sg5ktMLPV\nZubAafnDxA9hPQ6siil34KoKbq9gu4hIB8g6RnIM8O/A4cCZwNMkr07PO4T4le1GWNxYLgXbRUQ6\nQJYLEl8hrEZ/BLjE3X9NeqAdQqr5pPGmvyTcSyvbRUQ6SZY9kl9Fn/8GjDSzDcBYkhcb5g0i7KRY\nrBHYKuGaLVtVQxERabMsZ21NJkzl/VH0eSIwqYzrhhIfO8kBaxOuWdiaCoqISNtludXu9oSpvJ+J\n8mzVR4fTcm0tI3lB4u0J5b0qrmREs7ZERNomy612FxIahM+b2TTgFELjkrYfyRCSFyR+JqE8bshL\ns7ZERDpAljGSgdHrLwB2JKSSh+TFhnkPAbNjyhtICMK7+4yYYs3aEhHpAFnGSAYThrO2BP5M2PVw\nJCHgXqob8AnCbK8457dnBUVEpO2yjJHkCLGLRkKyxoMIw1Zp3k0oz5GwKVaUgkVERDpBljESgMXA\nGWyKjQwjfWhrZ+LzcRnJ2X/TMgqLiEhGsuyRQBjKuhZ4K7rX+8CGlMvnk7xw8eGE8o/ElJUVbNes\nLRGRtumIHslngbOAeWVevjXJsZstEsrXxJSVFWzXrC0RkbbJukcyjNCQzCM0EMOIX7VeaDUJsRCS\n4ydxs7ZERKQDZN0jOQi4gbAXCaSvIcmfkxTziE0j7+6LKqqgiIi0myzXkTQAKwp2R3yKMA14Zcp1\nScNXAAfEFZqZgu0iIp0ky3UkBjxlZn0JiRrLbbRWEwLlxY2DA3smXDMcWNqaSirYLiLSNln2SBoJ\nSRY/DhwFTAdeJwTgSxlE/NCWkbyXybKYMqVIERHpAJn0SKJgezUwDvgjIcDel9BwpTVea4H+MeWN\nhEapBXePC86vJvRUREQkQ1kG298lrAnZzd0HAz8FHiN9CCquEYGwtqRVw1ciIpKdrLfajTMs5fhb\nwJsx5Qa83P7VERGRtsgy2J6kPuW+IxPKq0hPryIiIh0sy4ZkICED8NtmVkfY+dBJXmyY58CAhGOV\nRMaVIkVEpANkObQ1iBAgP5lNjYeVcc/XShxLip/EUYoUEZEOkOWsrRyh4fhf4GLCQsMDCT2FA0ie\nnbULoeEpngJswH3Av8XczxJmbrWQy+VobCxngb2IiJQjy1lb8wiztvZw9ysLDu8bfU7qXSTNzDLC\n5lhxNM1XRKSTdMasrfwuhy8lHO9Lck6uzyWUv9+mGomISKt1xqytvGWEler9isqfIcRXihcf1gNv\nx5RD6JEsKSq7FDgdYPny5btNnDhxZlS+T+FJixcv3jhx4sSkRq2rGk78av7urKc9k56n6+tpz5TF\n84wr56TObEheAvaPKb8OODum/ANCrOXkmGMthsPc/brotTCzf7r7vtHXG2m+A+M7+WPdReHz9BQ9\n7Zn0PF1fT3umznyejmhIRpjZy8AoQpzj36Py6cD9wOcL6uHAc8B2Ma9zOHA0UEfYC35j/rpyA+0i\nItL+MouRuPu27r6Nu7/q7qPdPefu5u75ey4mTNEtbMwMGObuOwLPFpRvJAxJ/ZIw28sIjQnAtyqs\nV3VUj/zH+FY8noiIRDplaMvd81N7HwbOSDgndu8RwsLGSl3Ximu6sp72PNDznknP0/X1tGfqtOcx\njQqJiEhbdMb0XxER6UHUkIiISJv0+IbEzD5pZrPN7A0zO6+z69MaZvaOmc0ws+lm9s+obKiZPWJm\nr0efh3R2PZOY2U1mtsTMZhaUxdbfgl9E79fLZrZ359U8WcIzTTGzBdH7NN3Mji44dn70TLPN7MjO\nqXUyMxtrZo+Z2Swze8XMzo7Ku+X7VOJ5uuV7ZGY1Zvacmb0UPc/UqHw7M3s2en9+Z2a9o/I+0fdv\nRMe3zbSC7t5jPwjrRd4Etgd6E9auTOjserXiOd4BhheV/Qw4L/r6POCyzq5nifp/DNgbmJlWf8IU\n7z8RZuYdADzb2fWv4JmmAN+NOXdC9G+vD2Fq+5tAVWc/Q1EdtwL2jr4eCMyJ6t0t36cSz9Mt36Po\n5zwg+roXYVbrAcCdwKSo/Frg69HXk4Fro68nAb/Lsn49vUeyP/CGu7/l7nXAb4FjO7lO7eVY4NfR\n178GJnZiXUpy9yeA5UXFSfU/FrjVg2eAwWa2VcfUtHwJz5TkWOC37r7B3d8G3iB+MW6ncfdF7v6v\n6OvVwCxga7rp+1TieZJ06fco+jmvib7tFX048Ang7qi8+P3Jv293A4eZWXEi3HbT0xuSrQnJI/Pm\nU/ofU1flwMNm9oKZnR6VjXT3RRD+0wAjOq12rZNU/+7+np0ZDfXcVDDc2K2eKRoG2YvwV2+3f5+K\nnge66XtkZlVmNp2QDuoRQq9ppbtvjE4prHPT80THPyB9d9pW6+kNSVwL3B3nO3/U3fcGjgK+YWYf\n6+wKZag7v2f/D/gQsCewCLgiKu82z2RmA4BpwDnuvqrUqTFlXe6ZYp6n275H7t7g7nsCYwi9pV3i\nTos+d+jz9PSGZD4wtuD7McDCTqpLq7n7wujzEuD3hH9Ei/NDCdHn4qSVXV1S/bvte+bui6P/7I3A\n9WwaGukWz2RmvQi/dG9393ui4m77PsU9T3d/jwDcfSXwOCFGMtjM8gvLC+vc9DzR8S0ofyi2Yj29\nIXkeGB/NbOhNCDrd18l1qoiZ9TezgfmvgSOAmYTnOCk67STgD51Tw1ZLqv99wInRrKADgA/yQytd\nXVGM4DjC+wThmSZFM2m2A8YTcsp1GdH4+Y3ALHf/n4JD3fJ9Snqe7voemdmWZjY4+rovIWfhLOAx\n4LPRacXvT/59+yzwV48i75no7NkIWX8QZpfMIYwnXtDZ9WlF/bcnzCZ5CXgl/wyE8c5Hgdejz0M7\nu64lnuEOwjBCPeEvpVOS6k/okl8TvV8zgH07u/4VPNNvojq/TPiPvFXB+RdEzzQbOKqz6x/zPAcR\nhj5eJiRUnR793+mW71OJ5+mW7xGwB/BiVO+ZwIVR+faEBu8N4C6gT1ReE33/RnR8+yzrpxQpIiLS\nJj19aEtERDKmhkRERNpEDYmIiLSJGhIREWkTNSQiItImnbJDokhPYWYNhOmkeRPd/Z1Oqo5Ip9D0\nX5E2MLM17j6gA+9X7ZtyK4l0CRraEsmQmW1lZk9Ee1/MNLODo/JPmtm/ov0lHo3KhprZvVFCwWfM\nbI+ofIqZXWdmDwO3Rsn7Ljez56Nz/7MTH1FEQ1sibdQ3ysgK8La7H1d0/EvAQ+5+sZlVAf3MbEtC\nnqePufvbZjY0Oncq8KK7TzSzTwC3EpILAuwDHOTutVEG6A/cfT8z6wM8bWYPe0h/LtLh1JCItE2t\nh4ysSZ4HbooSCN7r7tPN7FDgifwvfnfPJ9M7CPiPqOyvZjbMzLaIjt3n7rXR10cAe5hZPsfSFoTc\nUGpIpFOoIRHJkLs/EaX9/xTwGzO7HFhJfErvUqm/1xad9013f6hdKyvSSoqRiGTIzMYBS9z9ekI2\n2r2BfwCHRFlmKRjaegL4clR2KLDM4/cEeQj4etTLwcx2jDJDi3QK9UhEsnUocK6Z1QNrgBPdfWkU\n57jHzHKEPT4OJ+wnfrOZvQysY1Ma8GI3ANsC/4rSpS+lC2+1LD2fpv+KiEibaGhLRETaRA2JiIi0\niRoSERFpEzUkIiLSJmpIRESkTdSQiIhIm6ghERGRNvn/DZcl02RmXD4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x150d4eb7780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x150dc22ae80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from xgboost import plot_importance\n",
    "from matplotlib import pyplot\n",
    "\n",
    "plot_importance(XGmodel)\n",
    "pyplot.figure(figsize = (1200, 1200))\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "       Fake       0.94      0.96      0.95      1071\n",
      "       Real       0.95      0.94      0.95      1020\n",
      "\n",
      "avg / total       0.95      0.95      0.95      2091\n",
      "\n",
      "F1 score: 0.947160493827\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "y_pred1 = XGmodel.predict(X_test)\n",
    "target_names = ['Fake', 'Real']\n",
    "print(classification_report(y_test, y_pred1, target_names=target_names))\n",
    "f1 = metrics.f1_score(y_test, y_pred1)\n",
    "print('F1 score:', f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best Random Forrests Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "Not Admitted       0.80      0.94      0.86      1071\n",
      "    Admitted       0.92      0.76      0.83      1020\n",
      "\n",
      " avg / total       0.86      0.85      0.85      2091\n",
      "\n",
      "F1 score: 0.829924650161\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rdf = RandomForestClassifier(n_estimators=10, criterion='gini', max_depth=None, min_samples_split=2, min_samples_leaf=1,\n",
    "                             min_weight_fraction_leaf=0.0, max_features='auto', max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
    "                             min_impurity_split=None, bootstrap=True,\n",
    "                             oob_score=False, n_jobs=1, random_state=None, verbose=0, warm_start=False, class_weight=None)\n",
    "rdf = clf.fit(X_train, y_train)\n",
    "y_pred2 = clf.predict(X_test)\n",
    "rdf.score(X_test, y_test)\n",
    "\n",
    "print(classification_report(y_test, y_pred2, target_names = target_names))\n",
    "f1 = metrics.f1_score(y_test, y_pred2)\n",
    "print('F1 score:', f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best MLP Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "Not Admitted       0.93      0.92      0.93      1071\n",
      "    Admitted       0.92      0.93      0.92      1020\n",
      "\n",
      " avg / total       0.92      0.92      0.92      2091\n",
      "\n",
      "F1 score: 0.922776148583\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "MLP = MLPClassifier(hidden_layer_sizes=(100, ), activation='relu', solver='adam', alpha=0.0001, batch_size='auto',\n",
    "                    learning_rate='constant', learning_rate_init=0.001, power_t=0.5, max_iter=200, shuffle=True,\n",
    "                    random_state=None, tol=0.0001, verbose=False, warm_start=False, momentum=0.9, nesterovs_momentum=True,\n",
    "                    early_stopping=False, validation_fraction=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-08)\n",
    "MLP.fit(X_train, y_train)\n",
    "\n",
    "y_pred3 = MLP.predict(X_test)\n",
    "print(classification_report(y_test, y_pred3, target_names = target_names))\n",
    "f1 = metrics.f1_score(y_test, y_pred3)\n",
    "print('F1 score:', f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best SVM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "Not Admitted       0.80      0.94      0.86      1071\n",
      "    Admitted       0.92      0.76      0.83      1020\n",
      "\n",
      " avg / total       0.86      0.85      0.85      2091\n",
      "\n",
      "F1 score: 0.829924650161\n"
     ]
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "Svm = svm.SVC(C=1.0, kernel='rbf', degree=3, gamma='auto', coef0=0.0, shrinking=True, probability=False,\n",
    "              tol=0.001, cache_size=200,\n",
    "              class_weight=None, verbose=False, max_iter=-1, decision_function_shape='ovr', random_state=None)\n",
    "Svm.fit(X_train, y_train)\n",
    "\n",
    "y_pred4 = clf.predict(X_test)\n",
    "Svm.score(X_test, y_test)\n",
    "\n",
    "print(classification_report(y_test, y_pred4, target_names = target_names))\n",
    "f1 = metrics.f1_score(y_test, y_pred4)\n",
    "print('F1 score:', f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best Logistic Regression Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "Not Admitted       0.90      0.93      0.92      1071\n",
      "    Admitted       0.92      0.90      0.91      1020\n",
      "\n",
      " avg / total       0.91      0.91      0.91      2091\n",
      "\n",
      "F1 score: 0.909000497265\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "Logistic_model = LogisticRegression(penalty='l2', dual=False, tol=0.0001, C=1.0, fit_intercept=True, intercept_scaling=1, class_weight=None,\n",
    "                                    random_state=None,\n",
    "                                    solver='liblinear', max_iter=100, multi_class='ovr', verbose=0, warm_start=False, n_jobs=1)\n",
    "Logistic_model = model.fit(X_train, y_train)\n",
    "\n",
    "y_pred5 = model.predict(X_test)\n",
    "\n",
    "Logistic_model.score(X_test, y_test)\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "target_names = ['Not Admitted', 'Admitted']\n",
    "\n",
    "print(classification_report(y_test, y_pred5, target_names=target_names))\n",
    "f1 = metrics.f1_score(y_test, y_pred5)\n",
    "print('F1 score:', f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vote Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def vote_func_5(X_test):\n",
    "    \n",
    "    y_pred1 = XGmodel.predict(X_test)\n",
    "    y_pred2 = rdf.predict(X_test)\n",
    "    y_pred3 = Svm.predict(X_test)\n",
    "    y_pred4 = Logistic_model.predict(X_test)\n",
    "    y_pred5 = MLP.predict(X_test)\n",
    "    \n",
    "    y_pred = []\n",
    "    for i in range(len(y_pred1)):\n",
    "        if y_pred1[i] + y_pred2[i] + y_pred3[i] + y_pred4[i] + y_pred5[i] >= 3:\n",
    "            y_pred.append(1)\n",
    "        else:\n",
    "            y_pred.append(0)\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def vote_func_3(X_test):\n",
    "    \n",
    "    y_pred1 = XGmodel.predict(X_test)\n",
    "    y_pred2 = Logistic_model.predict(X_test)\n",
    "    y_pred3 = MLP.predict(X_test)\n",
    "    \n",
    "    y_pred = []\n",
    "    for i in range(len(y_pred1)):\n",
    "        if y_pred1[i] + y_pred2[i] + y_pred3[i] >= 2:\n",
    "            y_pred.append(1)\n",
    "        else:\n",
    "            y_pred.append(0)\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def vote_func_2(X_test):\n",
    "    \n",
    "    y_pred1 = XGmodel.predict(X_test)\n",
    "    y_pred2 = MLP.predict(X_test)\n",
    "    \n",
    "    y_pred = []\n",
    "    for i in range(len(y_pred1)):\n",
    "        if y_pred1[i] + y_pred2[i] >= 1:\n",
    "            y_pred.append(1)\n",
    "        else:\n",
    "            y_pred.append(0)\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Best five models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "Not Admitted       0.91      0.95      0.93      1071\n",
      "    Admitted       0.94      0.90      0.92      1020\n",
      "\n",
      " avg / total       0.92      0.92      0.92      2091\n",
      "\n",
      "F1 score: 0.918213748118\n"
     ]
    }
   ],
   "source": [
    "y_pred = vote_func_5(X_test)\n",
    "print(classification_report(y_test, y_pred, target_names=target_names))\n",
    "f1 = metrics.f1_score(y_test, y_pred)\n",
    "print('F1 score:', f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Best three models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "Not Admitted       0.93      0.94      0.94      1071\n",
      "    Admitted       0.94      0.92      0.93      1020\n",
      "\n",
      " avg / total       0.93      0.93      0.93      2091\n",
      "\n",
      "F1 score: 0.931683168317\n"
     ]
    }
   ],
   "source": [
    "y_pred = vote_func_3(X_test)\n",
    "print(classification_report(y_test, y_pred, target_names=target_names))\n",
    "f1 = metrics.f1_score(y_test, y_pred)\n",
    "print('F1 score:', f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Best two models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "Not Admitted       0.96      0.91      0.94      1071\n",
      "    Admitted       0.91      0.96      0.94      1020\n",
      "\n",
      " avg / total       0.94      0.94      0.94      2091\n",
      "\n",
      "F1 score: 0.935622317597\n"
     ]
    }
   ],
   "source": [
    "y_pred = vote_func_2(X_test)\n",
    "print(classification_report(y_test, y_pred, target_names=target_names))\n",
    "f1 = metrics.f1_score(y_test, y_pred)\n",
    "print('F1 score:', f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "(1) The best size for TF-IDF is around 500  \n",
    "(2) The best F1-rate we get right now is around 0.95  \n",
    "(3) The Vote method didn't help as much as we thought it would be  \n",
    "(4) The best model we use is still XGBoost  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}