chapter6.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "newsdf=pd.read_csv(\"src/newsCorpora.csv\",header=None,sep=\"\\t\", names=['ID', 'TITLE', 'URL', 'PUBLISHER', 'CATEGORY', 'STORY', 'HOSTNAME', 'TIMESTAMP'])\n",
    "newsdf=newsdf[newsdf.PUBLISHER.isin([\"Reuters\",\"Huffington Post\",\"Businessweek\",\"Contactmusic.com\",\"Daily Mail\"])][[\"CATEGORY\",\"TITLE\"]]\n",
    "train,test=train_test_split(newsdf,test_size=0.2,stratify=newsdf.CATEGORY)\n",
    "valid,test=train_test_split(test,test_size=0.5,stratify=test.CATEGORY)\n",
    "train.to_csv('src/train.txt', index=False, sep=\"\\t\",header=None)\n",
    "valid.to_csv('src/valid.txt', index=False, sep=\"\\t\",header=None)\n",
    "test.to_csv('src/test.txt', index=False, sep=\"\\t\",header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TITLE</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CATEGORY</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>b</th>\n",
       "      <td>5627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e</th>\n",
       "      <td>5279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m</th>\n",
       "      <td>910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t</th>\n",
       "      <td>1524</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          TITLE\n",
       "CATEGORY       \n",
       "b          5627\n",
       "e          5279\n",
       "m           910\n",
       "t          1524"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newsdf.groupby(\"CATEGORY\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b    5627\n",
       "e    5279\n",
       "t    1524\n",
       "m     910\n",
       "Name: CATEGORY, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newsdf.CATEGORY.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 51"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TF-IDFというのがあるらしい  \n",
    "nltkを使うやつ：時間かかりすぎるので断念"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import string\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalizeText(text):\n",
    "    text = text.lower()  # 小文字化\n",
    "    table = str.maketrans(string.punctuation, ' '*len(string.punctuation))\n",
    "    text = re.sub('[0-9|,|.]+', '0', text)\n",
    "    text = text.translate(table)\n",
    "    text = re.sub(r\"\\s+\", \" \", text)\n",
    "    text = re.sub(r\"^ | $\", \"\", text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "Tfidfvec = TfidfVectorizer(min_df=5, max_df=0.9)#,max_features=1000, ngram_range=(1, 2))\n",
    "\n",
    "train_vec=Tfidfvec.fit_transform(train.TITLE.map(lambda x: normalizeText(x)))\n",
    "valid_vec=Tfidfvec.transform(valid.TITLE.map(lambda x: normalizeText(x)))\n",
    "test_vec=Tfidfvec.transform(test.TITLE.map(lambda x: normalizeText(x)))\n",
    "\n",
    "train_feature=pd.DataFrame(train_vec.toarray(), columns=Tfidfvec.get_feature_names())\n",
    "valid_feature=pd.DataFrame(valid_vec.toarray(), columns=Tfidfvec.get_feature_names())\n",
    "test_feature=pd.DataFrame(test_vec.toarray(), columns=Tfidfvec.get_feature_names())\n",
    "\n",
    "# train_feature.to_csv(\"src/train.feature.txt\", index=False, sep=\"\\t\")\n",
    "# valid_feature.to_csv(\"src/valid.feature.txt\", index=False, sep=\"\\t\")\n",
    "# test_feature.to_csv(\"src/test.feature.txt\", index=False, sep=\"\\t\")\n",
    "# train_feature=pd.read_csv(\"src/train.feature.txt\",sep=\"\\t\")\n",
    "# valid_feature=pd.read_csv(\"src/valid.feature.txt\",sep=\"\\t\")\n",
    "# test_feature=pd.read_csv(\"src/test.feature.txt\",sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 52"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(max_iter=10000)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_Log = LogisticRegression( max_iter=10000)\n",
    "clf_Log.fit(train_feature, train['CATEGORY'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 53"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_res=pd.DataFrame(np.array([np.max(clf_Log.predict_proba(train_feature),axis=1),clf_Log.predict(train_feature)]).T,columns=[\"predict_prob\",\"predict_category\"])\n",
    "test_res=pd.DataFrame(np.array([np.max(clf_Log.predict_proba(test_feature),axis=1),clf_Log.predict(test_feature)]).T,columns=[\"predict_prob\",\"predict_category\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 54"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9418103448275862\n",
      "0.8905547226386806\n"
     ]
    }
   ],
   "source": [
    "#カットなし\n",
    "print(clf_Log.score(train_feature,train.CATEGORY))\n",
    "print(clf_Log.score(test_feature,test.CATEGORY))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9173538230884558\n",
      "0.8860569715142429\n"
     ]
    }
   ],
   "source": [
    "#10以下カット\n",
    "print(clf_Log.score(train_feature,train.CATEGORY))\n",
    "print(clf_Log.score(test_feature,test.CATEGORY))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9421851574212894\n",
      "0.8740629685157422\n"
     ]
    }
   ],
   "source": [
    "#n2gramも含めたけど…\n",
    "print(clf_Log.score(train_feature,train.CATEGORY))\n",
    "print(clf_Log.score(test_feature,test.CATEGORY))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9196026986506747\n",
      "0.8883058470764618\n"
     ]
    }
   ],
   "source": [
    "#n2 10以下カット\n",
    "print(clf_Log.score(train_feature,train.CATEGORY))\n",
    "print(clf_Log.score(test_feature,test.CATEGORY))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9418103448275862\n",
      "0.8905547226386806\n"
     ]
    }
   ],
   "source": [
    "#n2 5以下カット\n",
    "print(clf_Log.score(train_feature,train.CATEGORY))\n",
    "print(clf_Log.score(test_feature,test.CATEGORY))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 55"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'clf_Log' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m-----------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m           Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-5376035e66ef>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtrain_true\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCATEGORY\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrain_pred\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclf_Log\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_feature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mtest_true\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCATEGORY\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mtest_pred\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclf_Log\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_feature\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'clf_Log' is not defined"
     ]
    }
   ],
   "source": [
    "train_true=train.CATEGORY\n",
    "train_pred=clf_Log.predict(train_feature)\n",
    "test_true=test.CATEGORY\n",
    "test_pred=clf_Log.predict(test_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4360,   78,    7,   57],\n",
       "       [  33, 4181,    1,    8],\n",
       "       [  84,  133,  504,    7],\n",
       "       [ 177,  119,    3,  920]])"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#train confusion\n",
    "confusion_matrix(train_true, train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[538,  18,   1,   6],\n",
       "       [ 18, 508,   0,   2],\n",
       "       [ 14,  21,  53,   3],\n",
       "       [ 35,  25,   2,  90]])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#test confusion\n",
    "confusion_matrix(test_true, test_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 56"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           b       0.94      0.97      0.95      4502\n",
      "           e       0.93      0.99      0.96      4223\n",
      "           m       0.98      0.69      0.81       728\n",
      "           t       0.93      0.75      0.83      1219\n",
      "\n",
      "    accuracy                           0.93     10672\n",
      "   macro avg       0.94      0.85      0.89     10672\n",
      "weighted avg       0.93      0.93      0.93     10672\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#train\n",
    "print(classification_report(train_true, train_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           b       0.89      0.96      0.92       563\n",
      "           e       0.89      0.96      0.92       528\n",
      "           m       0.95      0.58      0.72        91\n",
      "           t       0.89      0.59      0.71       152\n",
      "\n",
      "    accuracy                           0.89      1334\n",
      "   macro avg       0.90      0.77      0.82      1334\n",
      "weighted avg       0.89      0.89      0.88      1334\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#test\n",
    "print(classification_report(test_true, test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#recallが低い：あるカテゴリーをそのカテゴリーだと識別できた割合が低い"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 57"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "coefs=pd.DataFrame(clf_Log.coef_.T,columns=clf_Log.classes_)\n",
    "coefs[\"terms\"]=Tfidfvec.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "business\n",
      "top  : china\tfed\tbank\tstocks\tecb\teuro\tupdate\tukraine\toil\tdollar\n",
      "worst: and\tthe\tebola\ther\tshe\tgoogle\tfacebook\tapple\tvideo\tstudy\n",
      "entertainment\n",
      "top  : kardashian\tchris\tkim\tstar\tmiley\ther\tcyrus\tpaul\tshe\tmovie\n",
      "worst: update\tus\tgoogle\tsays\tchina\tfacebook\tstudy\tceo\tgm\tapple\n",
      "health\n",
      "top  : ebola\tcancer\tstudy\tfda\tdrug\tmers\thealth\tcigarettes\tcdc\tcould\n",
      "worst: gm\tfacebook\tgoogle\tapple\tceo\tfed\tclimate\ttwitter\tdeal\tbank\n",
      "science and technology\n",
      "top  : google\tfacebook\tapple\tmicrosoft\tclimate\ttesla\tgm\tnasa\tfcc\tsamsung\n",
      "worst: stocks\tfed\tpercent\tamerican\tdrug\ther\tcancer\tecb\tkardashian\tshares\n"
     ]
    }
   ],
   "source": [
    "category={\"b\":\"business\",\"t\":\"science and technology\",\"e\":\"entertainment\",\"m\":\"health\"}\n",
    "for i in clf_Log.classes_:\n",
    "    print(category[i])\n",
    "    print(\"top  :\",\"\\t\".join(coefs.sort_values(i,ascending=False)[:10][\"terms\"].values))\n",
    "    print(\"worst:\",\"\\t\".join(coefs.sort_values(i,ascending=True)[:10][\"terms\"].values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 58"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "Tfidfvec = TfidfVectorizer(min_df=10, max_df=0.8)#,max_features=1000, ngram_range=(1, 2))\n",
    "\n",
    "train_vec=Tfidfvec.fit_transform(train.TITLE.map(lambda x: normalizeText(x)))\n",
    "valid_vec=Tfidfvec.transform(valid.TITLE.map(lambda x: normalizeText(x)))\n",
    "test_vec=Tfidfvec.transform(test.TITLE.map(lambda x: normalizeText(x)))\n",
    "\n",
    "train_feature=pd.DataFrame(train_vec.toarray(), columns=Tfidfvec.get_feature_names())\n",
    "valid_feature=pd.DataFrame(valid_vec.toarray(), columns=Tfidfvec.get_feature_names())\n",
    "test_feature=pd.DataFrame(test_vec.toarray(), columns=Tfidfvec.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_true=train.CATEGORY\n",
    "valid_true=valid.CATEGORY\n",
    "test_true=test.CATEGORY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [07:23<00:00, 55.42s/it]\n"
     ]
    }
   ],
   "source": [
    "scores=[]\n",
    "for C in tqdm(np.logspace(-3, 4, 8, base=10)):\n",
    "    clf_Log = LogisticRegression(max_iter=10000, C=C)\n",
    "    clf_Log.fit(train_feature, train_true)\n",
    "    train_score=clf_Log.score(train_feature, train_true)\n",
    "    valid_score=clf_Log.score(valid_feature, valid_true)\n",
    "    test_score=clf_Log.score(test_feature, test_true)\n",
    "    scores.append([train_score,valid_score,test_score])\n",
    "scores=np.array(scores).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9f3v8ddnsifsq0CAACKKyBpQBIRaF9zqglZxa62Waqv+utja7f76u23vr72tvV1tLVhr3UpFrSLi9lNZrFiSIKsIhLAkYSds2Wf53j8mYgxZJjCTk5m8n4/HPDJnmZn3nEzec3LOmTPmnENEROKfz+sAIiISHSp0EZEEoUIXEUkQKnQRkQShQhcRSRAqdBGRBJHs1QP36tXL5eTkePXwIiJxqaCg4IBzrndj0zwr9JycHPLz8716eBGRuGRmO5qapk0uIiIJQoUuIpIgVOgiIglChS4ikiBaLHQze8zM9pnZ+iamm5n9zswKzWytmY2PfkwREWlJJGvojwMzm5l+GTC87jIH+NOpxxIRkdZq8bBF59wyM8tpZpargSdc+Dy875tZNzPr55zbHaWMIgkhGHIEQw6HwzlwDkLO4aj76cDV/TxxfL1xofApr0MN5nXOEao/b93PpuYNDzd47PqP2chzaOx0202egLuRCa6RkU2dwbux8Ylysu+cnpkM79s56vcbjePQBwDF9YZL6sadUOhmNofwWjyDBg2KwkOLtG/OOVYUHWTesiKWbN7fZHlJx3L39GF897Izo36/0Sh0a2Rcoy9b59xcYC5Abm6uXtqSsPzBEIvX7Wbe8iLWlx6lV6dUvjxtKF0zUgDwmWEGPgMjfN3M6obB5zPMLHy9uXktPP3T9xm+nTUY9vlo5D4bmbcug1ljf9rhxzxhXBPLobH7aGzeJh4Ka2TupuaNJ706pcXkfqNR6CXAwHrD2cCuKNyvSNw5Vu3nH3nFPPbuNnYdqWZY7yx+ft05XDNuAOkpSV7HkwQXjUJfCNxrZvOBc4Ej2n4uHc3uI1U8/q/tPPPvnRyrCXDe0B785JpRfGZEH3y+BFillLjQYqGb2d+BGUAvMysBfgSkADjnHgEWA5cDhUAlcEeswoq0Nx/uOsqjy4tYuGYXDrj8nH58edoQRmd38zqadECRHOUyu4XpDvha1BKJtHPOOZZtOcCjy4tYvuUAmalJ3D45hzum5DCwR6bX8aQD8+xsiyLxpjYQYuGaXTy6vIiP9hyjT+c0Hpx5JjdPGkTXzBSv44mo0EVacqTKzzP/3snj721j79EaRvTtzEM3jOFzY/qTmqyzZ0j7oUIXaUJxWSV//dd2/pG3k4raIFNP78Uvrh/DBcN7NXlIn4iXVOgiDawtOczcZUW8un4PBnxuTH/unDaEs/t39TqaSLNU6CKEP07/zqZ9zF1WxL+3ldE5LZm7pg7hi1Ny6Nc1w+t4IhFRoUuHVu0P8uIHpcxbXsTW/RX075rOD684ixsnDqRzunZ0SnxRoUuHdKiilqfe38HfVmznQHktZ/fvwm9vGsvl5/QjJUk7OiU+qdClQ9l+oIK/vLuNBQXFVPtDzBjRmznThjJ5WE/t6JS4p0KXDqFgxyHmLSvi9Q/3kOLzcc24/tw1bShnxOAUpiJeUaFLwgqGHG9+uJd5y4so2HGIrhkpfHXGML4wOYc+XdK9jicSdSp0SThVtUGeW1XCX5YXsf1gJdndM/ivq0ZyQ+5AstL0kpfEpVe3JIz9x2p4csV2nnx/B4cq/YzJ7srDN4/n0rP7kqwdndIBqNAl7hXuK+cv7xbx/KpSagMhLjqrL3MuGMrEnO7a0Skdigpd4tbq4sP84e0t/M/GfaQm+5g1Ppu7pg1hWO9OXkcT8YQKXeLSBzsPcePc98lKTeL+zw7n9smDY/a1XiLxQoUucWfPkWq+8mQBfbuk8dLXptIjK9XrSCLtgvYUSVypqg0y58l8KmoCPHr7RJW5SD1aQ5e44ZzjO8+vZV3pEebdlsuI0/ShIJH6VOgSNx5+p5CX1+ziOzNHcNHIvl7HiViFv4IVu1aw7sA6gqEgDkfIhQAIuVCjw8654+MbnddBiNDx+Zyrm5dPTzs+3Mj91r9dY9PiiQ8f6cnpZCRnHP+ZkZxBZnLmp4brX2/skp6cTnpSetweHaVCl7jw+oY9PPTGZq4Z2597pg/zOk6LdpXvYknxEpaWLCVvTx7+kJ9kSyYlKXwGR5/58OEDC183DJ/5jk8zDDM7Pr7htEbnNcNHE/NauPQ+vk+zevdbf9rH0+vmiRdBF6Q6UM3B6oNUB6qpClQdv9QEa1p1X4Y1WfyfenNISicjpd4bR73h5qan+lJjtmxV6NLubdx9lG/8YzVjsrvy81mj22XRBENB1h1Yx9KSpSwtWcqWQ1sAGNxlMLPPnM2MgTMY22csKb4onpLXOfBXgb8yfKmtBH9F3c+qetdbM70SaivC080gKQWSUusu9a83Nq7uenJaE7dLgaS0pm+XlArJzd13I/cbwWsh5EJUB6qpDFRSFag6ofA/Hm5pelWgiv2V+0+YVhuqbdWvzWc+7hx1J/ePv/9kf/NNUqFLu3awvIa7/pZP5/Rk5t6eS3pKkteRjqvwV/DervdYWryU5aXLKasuI8mSGNdnHA/kPsD07OnkdM0Jzxz0Q9kOqD0WLstPlevJFnIltHbTSHIGpGZCSt0lNRNSsiCrN3TLCF9PzYSUjE9yB2vrLnXXA7WfHuc/Um96TeO3Cbau9CLmq/cmcvz51H8emfhSMslMzSQzJRNSs06YHr5dT8gYdOLtkzPA1/yxI4FQ4Hjp13/jaPTNIVhNpb+S8X3Hx2RxqNCl3aoNhLjnqVUcKK9hwd2T6dsOTqhVWl7K0uKln9qU0jm1M1MHTGVG9gymDJhC17SuUHMMSvJg1TOwcwWU5NcVcAuS0sKFkppVr6AyIb0bdOnfSBHVFXJKxifXm5qektliOcWMc00XfUtvBA3fQIK1DS5+CFTXvVFWfPJmV1kG/pJP//cRye+goZQT3yTq/26SU7PolJJBp+NvGJkNfodZkNoFsupNz+gW/WWMCl3aKecc//nSelZuL+O3N41ldHZs/gBaUn9TypLiJRQeLgQgp0sON595M9MHTg9vSik/EC7ut/4PFL8Pe9aBC4H5oO8oGHcb9B8H6V0bL+yPL0kJ+idpFt6ckuzxYaahUF35f7xpqYnNTZFML99XN73qk3kD1ZHlOP9+uOQnUX96CfrqkXj3+HvbmZ9XzNc+M4yrxw5o08f+eFPKkuIlvFv67vFNKeP7jg9vShkwjZza2nCBv/do+OfhHeEbp2RCdi5c8G0YdB4MyIX0Lm2aX5rh84XfRFMzIatX9O8/FKr3JtDMm0OfkdF/bFTo0g4t27yfnyz6kItH9uVbF49ok8csLS8NH5VSvJS8vXkEQgG6pHZh6oCpTO9/PlOsE113r4U1r8KiH0HVofANs3qHi/vcr4R/njY6vLNOOiafD9I6hS8eiKjQzWwm8FsgCXjUOffzBtO7A48Bw4Bq4EvOufVRziodQNH+cu59ZhVn9O3Mr28ci88XmyNaPt6U8vGhhfU3pdxy+iymJ/dg3OE9JBeuhKV/DW/jBeg5HM68EgZNDhd4j6ERHWkh0hZaLHQzSwIeBi4GSoA8M1vonPuw3mzfB1Y75641szPr5v9sLAJL4jpS5eeuv+WTnORj3u25dIryl1GU15aHj0opWcrykuUcqjkU3pTSYyQP9L+IGRUVDC5dC2t+Gb6BLzm83XvSlz8p8Fj8my4SJZH8xUwCCp1zRQBmNh+4Gqhf6COBnwE45z4ysxwz6+uc2xvtwJKYAsEQ9/39A3aWVfL0XecysEdmVO635FhJ+Njw+ptSkrOYmjmAGa47U3ZvpkvRK+GZ07rAwElwzqxwgfcfH97WKhInIin0AUBxveES4NwG86wBrgPeNbNJwGAgG/hUoZvZHGAOwKBBg04ysiSin736Ecs27+dn153DuUN7nvT9NLkpJbkLt4Y6MX1/MWPLd5LMRugyoG7Nu27tu89I8LWf49xFWiuSQm9sA2HDTzP8HPitma0G1gEfAIETbuTcXGAuQG5ubnydLEJi5tm8Yv7y7ja+eH4Osye1/o3+U5tSipdyqPYISRgTQsl8+/ARpldUMDhQHC7sM6/7pMC7DYzBsxHxTiSFXgLUf+VnA7vqz+CcOwrcAWDhz2Vvq7uINCtvexk/eHEd04b34odXnNWq276ydREvfjSf/IPrCLgQXUIwrbKC6ZVVTKkJ0WXABBh9TbjAsyfG7MMcIu1FJIWeBww3syFAKXATcHP9GcysG1DpnKsF7gKW1ZW8SJNKDlVy95MFZHfP5A+zx0f8Rc4hF+KX7/4vnipayGC/n1srqpgeTGbsaRNJPqNuE0q/MeGPg4t0IC0WunMuYGb3Aq8TPmzxMefcBjO7u276I8BZwBNmFiS8s/TOGGaWBFBRE+DLTxRQGwwx7/ZcumZGdux2TbCG771xD2/uy+OWihq+nfsASUOmhw8n9Opj7SLtRETHhTnnFgOLG4x7pN71FcDw6EaTRBUKOb717Bo27TnKY1+cyOl9IvsQxpGaI9z/ym2sOraNByrh9usXYn3OjHFakfihT4pKm/vNW1t4bcMefnjFWcwY0Sei25SWl3LPopspqT7IL/1ZzLz1Jeh8WoyTisQXFbq0qUVrd/G7t7Zww4Rs7pw6JKLbbDiwga+9+kVq/ZXMTR5E7u3PQpq+fk6kIW10lDazvvQIDyxYw4TB3fnptaMi+qKK5TuXcMcrN5NaW86T3SaSe8tClblIE1To0ib2Havmy0/k0yMzlUdunUBacssf4Hnhw6e57537GFxTzVODb2DYtX/Via9EmqFNLhJz1f4gX3mygMOVfp67ZzK9Ozd/OKFzjj+u/AWPfPQUU6qq+dWEB8maeFcbpRWJXyp0iSnnHN//5zo+2HmYR24dz9n9uzY7vz/k58dvf5MXS5dwTUUN/3nRH0g549I2SisS31ToElNzlxXxwqpSvnHRGcwc1a/ZeSv8FXzz1Tt479BG7qkIcM/V/8AGjGujpCLxT4UuMfP2R3v5+WsfccU5/bj/s6c3O+++yn18bdEtbKnczf+uSeW62Yug++A2SiqSGFToEhNb9h7j/r+vZmS/Ljx0w5hmj2jZeqiQexbfxuHao/zB9WHqbc9DZo82TCuSGFToEnWHKmq564l80lOSmHd7LhmpTR/Rkr97Jfe/+RVS/TU83mkUI697AlLS2zCtSOLQYYsSVf5giK89s4rdh6uZe/sE+nfLaHLe1woXMueNu+hVU8XTp13KyBvmq8xFToHW0CWqfvzyh7y39SC/umEM4wd1b3Qe5xxPrP4TD639E+Ora/jdqK/SdcrX2zipSOJRoUvUPPX+Dp58fwdzLhjKrAnZjc4TDAX5xbs/4Jltr3BJZTX/Pe3npI2a1cZJRRKTCl2iYsXWg/zXwg18ZkRvHpzZ+BkQqwPVfPeNu3lrfwG3V9TyrSv+hm/w+W2cVCRxqdDllO08WMk9TxeQ0yuL380eR5LvxCNaDlUf4r5XbmPtse08WGXc+vlF0EtnXBaJJhW6nJJj1X7ueiIP5+DR23PpnH7iuVaKjxVzz6Kb2V1ziF/5O3PxbS9Bp8hOmysikVOhy0kLhhxfn7+arfsrePJLk8jplXXCPOv3r+Nrr91B0F/Jo6lDGDf7H5AW2RdaiEjr6LBFOWm/fH0Tb320jx9dNZLzT+91wvRlO97iS4tvJaOmgie7n8+42S+pzEViSIUuJ+WfH5TwyNKt3HLuIG4778SP6C/Y8AT3Lfk6Q2qqeGrobIZcMw+S9A+hSCzpL0xa7YOdh3jw+XWcN7QH//W5sz/1sX7nHL9//7+Zt3k+0yqreWjSD8mc8AUP04p0HCp0aZXdR6qY82QBfbuk8cdbJpCS9Mk/ef6gnx+9/R+8vGs5sypq+OElfyT59Is9TCvSsajQJWJVtUHmPFFAZU2Ap++aQo+s1OPTjtUe4xuv3sG/D2/i3oogc659Dus32sO0Ih2PCl0i4pzjO8+vZf2uI8y7LZcz+n7yvZ57K/by1UU3U1S1l5/WpHH1LS9B18Y/KSoisaNCl4g8/E4hL6/ZxYMzz+SikX2Pj99Stpl7Ft9Gub+ch+00zr/9ecjo5mFSkY5LR7lIi15bv4eH3tjMteMGcPf0ocfHryxdwRcW3Uio5iiPZ57D+bcuVpmLeCiiQjezmWa2ycwKzey7jUzvamYvm9kaM9tgZndEP6p4YePuo3zz2dWMGdiNn113zvEjWl7Z/AJf+Z+v0Kemkqf7X8mZNzwDyc1/+bOIxFaLm1zMLAl4GLgYKAHyzGyhc+7DerN9DfjQOXeVmfUGNpnZ08652pikljZxoLyGu/6WT+f0ZObdNoH0lCScczy26vf8Zv08cqtr+M3o++g6+T6vo4oIkW1DnwQUOueKAMxsPnA1UL/QHdDZwqtvnYAyIBDlrNKGagMhvvrUKg6U17Dg7sn06ZJOMBTkZ8u+xz92vMplFdX8dMZDpI682uuoIlInkkIfABTXGy4Bzm0wzx+AhcAuoDNwo3MuFJWE0uacc/znS+tZub2M388ex+jsblQFqnjw9a/wzoEPuKOilq9f+RS+QQ1fBiLipUgKvbFv93UNhi8FVgMXAsOAN81suXPu6KfuyGwOMAdg0KBBrU8rbeLx97YzP6+Yez9zOleN6U9ZdRn3LbqVdeU7+X6Vj9k3Loaew7yOKSINRLJTtAQYWG84m/CaeH13AC+4sEJgG3DCtxw45+Y653Kdc7m9e/c+2cwSQ8s27+cniz7kkpF9+ebFZ7Dz6E5ue+FzbDq2g18HujL7trdU5iLtVCRr6HnAcDMbApQCNwE3N5hnJ/BZYLmZ9QVGAEXRDCqxEwiG+HD3UVZuK+N3b23hjL6d+fWNY1m/fw33vnEnzl/Fo6mnM/aW+ZCa6XVcEWlCi4XunAuY2b3A60AS8JhzboOZ3V03/RHgJ8DjZraO8CaaB51zB2KYW05BtT/I6uLD5G0rY+X2MlbtOERFbRCAM/p2Yt7tuazc/TbfWfptevtr+FPv6Qy+8mGdLVGknYvoL9Q5txhY3GDcI/Wu7wIuiW40iZaj1X4Kth9i5fYy8raVsbbkCLXB8D7rEX07c+34AUzM6cGkIT3o1zWD+ev+ys8K/h9n19Tw+xFfoOf074M1titFRNoTrXIloH3Hqsnbdoi87WWs3FbGxj1HcQ6SfcaoAV354pQcJuX0IDenO90yPznBlj/o59f/+jGPFS5gRlU1//e8H5E57lYPn4mItIYKPc4559hZVsnKbWXkbS8jb/shth2oACAjJYlxg7px/4XDOXdID8YO6kZm6ie/8kp/Je/vXkVB6QpWlbzL2qNbqXZBbqyo4buX/pnkYRd69bRE5CSo0ONMKOTYtPfY8bXvldvK2HesBoCuGSlMzOnO7EkDmZjTg1EDun7qfOVHao7wzs4CVu14h4I9K9lYuZsADp9zjKj1c311NZPTT2PadX/HThvl1VMUkZOkQm/nagMh1pUeOV7g+dvLOFod/hDuaV3SOW9oTyYO6cGknB4M79MJn++Tbd17K/ayqvQ9Cra/ScGBdRT6DwOQ4hzn1NRwhx/Gdx7C2Ozz6TR4CgyYAJk9PHmeInLqVOjtTEVNgA92HmbltoOs3F7G6uLDVPvDOzCH9s7i8nP6Hd+Bmd094/jJspxz7Dy6g4Ktr1FQsoyCI1soDVUDkBkKMa66hsuSOjOhx9mMGjyDtEHnQ+8R4Evy7LmKSHSp0D12qKL2+Np33vYy1u86SjDk8BmM7N+F2ZMG1e3A7EHvzp+czTAYCrJp7yoKtixi1Z48CipLOEj40MPuwSDja4PcktGP8aflMmLIxSQPnAQZ3b16miLSBlTobWzX4arwtu+6Qwi37CsHIDXZx9jsbtw9fSgTc3owYXB3OqenHL+dP1DL6s0vU7DtDQoOrmN17UGO1W1d6RcIMNmlMb7LcCYMmMqQ02divc8En053L9KRqNBjbN+xat7auO/4DszSw1UAdEpLZsLg7lwzLnwM+OjsrqSnfLL5o/LYHlZseJGC4uWsOlrI2lAFNXWbV4b4g1ya0p3xPc9mQs5F9B92sda+RUSFHkvBkOP6P61gZ1klvTqlMjGnB3dOHcKkIT04q18Xkj7egRkKcXj3B7y3ZSGr9uZRUFHKRl+QoBk+5zgzlMQNmdnk9s1l3OlX0WPARK19i8gJVOgx9G7hAXaWVfKL60dzw4Ts4zswqT7Kno/+yaptb1JwcB2rassorFs7T3WOUUkZfKnLMCZkT2XMiGvp1GWAh89CROKFCj2GFuQX0z0jiasHHGPHv3/PquJlFBzZSgHVlKaEF32Wg7EZvbi85yjG51zMqGEzSUvJ8Di5iMQjFXqMHKn088aHe7ij/4+5dPFRDiaH18B7pCQxPnMIt/SdwITTr+KM08aT7NOvQUROnZokRhauKeV0W838zHLGp/bm3hE3MH7oZQzpNvSTTS8iIlGkQo+RZ/NLGNHnTUqAn176Zwb0POH7PkREokqFHgMf7TnK1tLdVA3fy/kpvVTmItImdOxbDCzIL+G8LovYm5zErBE3eh1HRDoIFXqU+YMhXvygFF+PNfQMwYxxX/Y6koh0ECr0KHv7o310q11LfnqAq3uOJSUpteUbiYhEgQo9yhbkl3BWz1cJmjHr3Ae8jiMiHYgKPYr2HavmX5t2srnzLs71dWZQ3zFeRxKRDkSFHkUvflDK+MxX2Z2cxKzh13kdR0Q6GBV6lDjnWJBfQnrPArqF4LMT7vU6koh0MCr0KFlTcoSqsrWsyqjlc91GkpqS7nUkEelgVOhRsiC/mLN7LCJgxqxJ3/A6joh0QCr0KKj2B1m0Zic7u5Qw3jIZOuA8ryOJSAekQo+C1zfsYUTSYkpSfFw/5Cqv44hIBxVRoZvZTDPbZGaFZvbdRqZ/28xW113Wm1nQzHpEP2779FxBCVk9/03nkOPiSV/3Oo6IdFAtFrqZJQEPA5cBI4HZZjay/jzOuV8658Y658YC3wOWOufKYhG4vSk9XMWWbR+wJrOaqzoPJz2tk9eRRKSDimQNfRJQ6Jwrcs7VAvOBq5uZfzbw92iEiwfPF5RwVteX8ZsxK/c+r+OISAcWSaEPAIrrDZfUjTuBmWUCM4HnTz1a+xcKOZ7L38G+7tsZTRpn5FzodSQR6cAiKfTGvl7HNTHvVcC/mtrcYmZzzCzfzPL3798facZ2a+X2MnrULmZnio/rB13qdRwR6eAiKfQSYGC94WxgVxPz3kQzm1ucc3Odc7nOudzevXtHnrKdWpBfQrfu75IVclx6nk7EJSLeiqTQ84DhZjbEzFIJl/bChjOZWVdgOvBSdCO2T+U1Ad7bsJr1WZVckZVDZkZ3ryOJSAfXYqE75wLAvcDrwEbgWefcBjO728zurjfrtcAbzrmK2ERtXxav3c3wTv+kxmdcP+6rXscREYnsO0Wdc4uBxQ3GPdJg+HHg8WgFa++ezd9BTbdCRoZSOGv45V7HERHRJ0VPRtH+cir3vcL2VGNWto5sEZH2QYV+Ep4rKKFX9yVkhByXT/6O13FERIAIN7nIJ4Ihx6ur1lLe7xiXZ/SnU6e+XkcSEQG0ht5qy7fsp3/yAqp9xvWjv+x1HBGR41TorbSgoITybh9xRsjHqDNneR1HROQ4FXorHK6spajwZYrSYFa/qZhPi09E2g81UissXLOLXl3fJC3kuHLyg17HERH5FO0UbYV/5q1nV5dDXJraly5dB3kdR0TkU7SGHqGNu4+SVvkMlT4f14/6gtdxREROoEKP0IL8Eqq7r2doEMaec5vXcURETqBCj0BtIMTK9YvYmu6Y1edc7QwVkXZJzRSBtz/aR9fMxaQ4x+e0M1RE2ikVegSey/+Ios77uSipO916Dvc6johIo1ToLdh3rJojex6nPMnHDWfd4nUcEZEmqdBb8M9Vpfi7rWZQwJE77i6v44iINEmF3gznHG+seo2tGUFm9RqHJemwfRFpv1TozVhdfJhUe4Fk57j6PJ0mV0TaNxV6M57N28LOLnuZYZ3p2fccr+OIiDRLhd6Ean+QLUV/4WiSccMZN3gdR0SkRSr0Jry+YQ/BTnkMCIQ4L1dfAi0i7Z8KvQkLV75FYaafa7uNwpeS7nUcEZEWqdAbUXKokoqKZ0hyjmvP/ZbXcUREIqJCb8SzeUWUdilhqsugT/Ykr+OIiEREhd5AKORYueFRDicbnz/9Gq/jiIhETIXewL+3lVGT/i59AyGmTPoPr+OIiERMhd7AgveXsjWjmms6DScprZPXcUREIhZRoZvZTDPbZGaFZvbdJuaZYWarzWyDmS2Nbsy2UV4TYNe+xzBgltbORSTOtHhyEjNLAh4GLgZKgDwzW+ic+7DePN2APwIznXM7zaxPrALH0surd7Kry3bOC6bQb8hnvI4jItIqkayhTwIKnXNFzrlaYD5wdYN5bgZecM7tBHDO7YtuzLbxRv6jHEo2bhxyuddRRERaLZJCHwAU1xsuqRtX3xlAdzNbYmYFZnZ7Y3dkZnPMLN/M8vfv339yiWNk6/5yDttb9AoEuWDyA17HERFptUgK3RoZ5xoMJwMTgCuAS4H/ZWZnnHAj5+Y653Kdc7m9e/duddhY+vuKFRRlVnBl+mCSM7p7HUdEpNUiOcF3CTCw3nA2sKuReQ445yqACjNbBowBNkclZYwFQ44Pt/2JUDfjxon3eR1HROSkRLKGngcMN7MhZpYK3AQsbDDPS8A0M0s2s0zgXGBjdKPGzpLNe9iVtYWJtUb28Mu8jiMiclJaXEN3zgXM7F7gdSAJeMw5t8HM7q6b/ohzbqOZvQasBULAo8659bEMHk0vvPdXDqbAg6ddDNbYFiYRkfYvou9Uc84tBhY3GPdIg+FfAr+MXrS2caiill1Vi+ieEeSiKfpWIhGJXx3+k6Lz/53PtqyjXJbUj5ROfb2OIyJy0jp8oa/Y+HuCZtyce7fXUURETkmHLvT1pYcpTtvAuBrH4LNmeR1HROSUdOhCf3L5ExxIcVzXbxr4OvSiEJEE0GFbrDYQorBsAYHIJYgAAAlASURBVF2DQS6b9j2v44iInLIOW+gL16ynKPMQF9OTtG6DvI4jInLKOmyhv1bwGwJm3DzuTq+jiIhERYcs9L1HqthmBYyuCTJ89K1exxERiYoOWeiPLpvPvtQQV/acBEkRfbZKRKTd63CF7pxjdckTdA6GuOaCRr98SUQkLnW4Ql9eWMTW9P1cGOxMRu8zvY4jIhI1Ha7QF7z3K/w+4/Nna9u5iCSWDrUBubImwMbaFYx0AUZP+rLXcUREoqpDraE//t6L7E0NcGnWaEhO8zqOiEhUdahCf3fzPDJDIT4/QztDRSTxdJhC37hnD5tSS5lRm06nAeO8jiMiEnUdptD/+s5D1PqMa06/wesoIiIx0SF2igaDIVaXv8MIF2DyNH0JtIgkpg6xhr6g4A12p9ZyYepwSM3yOo6ISEx0iEJ/dd3DZIRC3HjBg15HERGJmYQv9D1HD7HBt51p1Sn0HDrF6zgiIjGT8IX+57d/Q40PZmZf4XUUEZGYSvidou/vX8ww/Hz2sw94HUVEJKYSeg39zY3vUpJazXQG48vs7nUcEZGYSuhCf3blr0kLhZh13te9jiIiEnMJW+jHaspZE9rM1Cofg0bN9DqOiEjMRVToZjbTzDaZWaGZnXAiFDObYWZHzGx13eU/ox+1deYu+RNVPpje67Ng5nUcEZGYa3GnqJklAQ8DFwMlQJ6ZLXTOfdhg1uXOuStjkPGkLCl5npyQn8uv/p7XUURE2kQka+iTgELnXJFzrhaYD1wd21inJr94NduTK5gWPI20bn29jiMi0iYiKfQBQHG94ZK6cQ1NNrM1ZvaqmZ3d2B2Z2Rwzyzez/P37959E3Mg8vvyXpDjHZaPvjtljiIi0N5EUemMboF2D4VXAYOfcGOD3wIuN3ZFzbq5zLtc5l9u7d+/WJY1QdaCa/Op1TK0Mcc55n4/JY4iItEeRFHoJMLDecDawq/4MzrmjzrnyuuuLgRQz6xW1lK3w5PuPUZHkOLfTFPAl7EE8IiIniKTx8oDhZjbEzFKBm4CF9Wcws9PMwoeSmNmkuvs9GO2wkVi8+RkG+v1cfol2hopIx9LiUS7OuYCZ3Qu8DiQBjznnNpjZ3XXTHwGuB+4xswBQBdzknGu4WSbmNh3YRGHSEW6q6kH304a29cOLiHgqonO51G1GWdxg3CP1rv8B+EN0o7Xeo+/8gmTnmD78i15HERFpcwlzcq7aYC3/Ks/n/KoAkz/zRa/jiIi0uYTZa/ji6vkc84UYlzKBpJRUr+OIiLS5hCn059b/lf7+ABdN/47XUUREPJEQhb7zyHY2coAplZ3IOWOM13FERDyREIX+6NKH8DlHbvZNXkcREfFM3Be6P+Tn7YPvcn6lnwsu/YrXcUREPBP3hf7mRws54gsyirPplNXJ6zgiIp6J+0J/etWf6RMIMHmSvpVIRDq2uC700mMlrAvsZlp5GuMmTPM6joiIp+K60J9a8VvAMbLn1Zi+lUhEOri4LfRAKMDi0rc4r6qWqZfc53UcERHPxW2hLyt6nTKfn5G1p9O/jydn6hURaVfittCfyvsjPQNBRp/zVa+jiIi0C3FZ6Hsq9lBQs5MLypM4f+pMr+OIiLQLcVnoz+b/kZDB0IxLSE9NmBNGioickrgr9GAoyIvbFjOpsoYJF2pnqIjIx+Ku0FfsXMJ+q+HMqmxGDRvsdRwRkXYj7go9tfADbjx6jNOH3qljz0VE6om7DdDlva/F/95hpn5hltdRRETalbgr9AvHDGP6OT8hyae1cxGR+uJukwugMhcRaURcFrqIiJxIhS4ikiBU6CIiCUKFLiKSIFToIiIJQoUuIpIgVOgiIgnCnHPePLDZfmAH0BU4Um9S/eGPrzf82Qs4cBIP2/CxIpkWSb6Wcp9M3uayNjW9uawtZWzrrJHmaym3lq2WrZdZm8rXUu5TyTvYOde70Tmcc55egLlNDX98vZGf+dF4rEimRZIvgtytzttc1qamN5c1lsv2ZLJq2WrZxuOybWmcl8vWOdcuNrm83Mzwy038jNZjRTItknxNXT+VvC3dtrHpzWVtOBzNZXsyWRsbr2UbWZZIpmvZtiwaWRuO83LZerfJ5VSYWb5zLtfrHJGKp7zxlBXiK288ZYX4yhtPWSF2edvDGvrJmOt1gFaKp7zxlBXiK288ZYX4yhtPWSFGeeNyDV1ERE4Ur2voIiLSgApdRCRBqNBFRBJEwhW6mZ1lZo+Y2XNmdo/XeVpiZteY2Twze8nMLvE6T3PMbKiZ/cXMnvM6S2PMLMvM/la3PG/xOk9L2vvybCjOXqvx1gNZZlZgZlee0h219uD2WF6Ax4B9wPoG42cCm4BC4LsR3pcP+Esc5e0ey7xRzvpce3xNALcBV9Vd/0dbZTzV5dyWyzNKeWP6Wo1y1pj3QDSyAj8GHgSuPKXH9eqF1MRCuAAYX38hAEnAVmAokAqsAUYC5wCLGlz61N3mc8B7wM3xkLfudr8CxsdJ1rYs9Nbk/h4wtm6eZ9r7a9iL5RmlvDF9rUYra1v1wKlmBS4CbgK+yCkWerv6kmjn3DIzy2kwehJQ6JwrAjCz+cDVzrmfAY3+e+KcWwgsNLNXgGfac14zM+DnwKvOuVXtOasXWpMbKAGygdV4tDmxlXk/bNt0J2pNXjPbSBu8VpvS2mXbVj0QhaydgCzC5V5lZoudc6GTedx42IY+ACiuN1xSN65RZjbDzH5nZn8GFsc6XCNalRe4j/A79PVmdncsgzWitcu2p5k9Aowzs+/FOlwzmsr9AjDLzP7EqZ8mIpoazduOlmdDTS1fL1+rTWlq2XrdA41pNKtz7gfOua8TftOZd7JlDrSvNfQmWCPjmvw0lHNuCbAkVmEi0Nq8vwN+F7s4zWpt1oNAe/hDbjS3c64CuKOtw0SgqbztZXk21FReL1+rTWkq6xK87YHGNPv35px7/FQfIB7W0EuAgfWGs4FdHmWJRDzljaes9cVbbuWNHWWtJx4KPQ8YbmZDzCyV8M6DhR5nak485Y2nrPXFW27ljR1lra+t9/62sGf478BuwE/43ezOuvGXA5sJ7yH+gdc54zFvPGWN59zKq6xeZtXJuUREEkQ8bHIREZEIqNBFRBKECl1EJEGo0EVEEoQKXUQkQajQRUQShApdRCRBqNBFRBKECl1EJEH8f8ZJ1/R/DbbEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cs=np.logspace(-3, 4, 8, base=10)\n",
    "plt.plot(cs,scores[0],label=\"train\")\n",
    "plt.plot(cs,scores[1],label=\"valid\")\n",
    "plt.plot(cs,scores[2],label=\"test\")\n",
    "plt.xscale(\"log\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "過学習がちょい見える"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 59"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "実行時間長すぎて断念  \n",
    "一応動くコード…"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_Log = LogisticRegression(C=10.0, max_iter=10000)\n",
    "clf_Log.fit(train_feature, train_true)\n",
    "train_pred=clf_Log.predict(train_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective(trial):\n",
    "    l1_ratio = trial.suggest_uniform('l1_ratio', 0, 1)\n",
    "    C = trial.suggest_loguniform('C', 1e-4, 1e4)\n",
    "    clf_Log=LogisticRegression(max_iter=10000,penalty=\"elasticnet\",solver='saga', l1_ratio=l1_ratio,C=C)\n",
    "    clf_Log.fit(train_feature, train_true)\n",
    "    valid_pred=clf_Log.predict(valid_feature)\n",
    "    return f1_score(valid_pred, valid_true, average='micro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2020-08-24 18:06:07,222] Trial 0 finished with value: 0.8808095952023988 and parameters: {'l1_ratio': 0.6575925074474871, 'C': 3.1044375292893234}. Best is trial 0 with value: 0.8808095952023988.\n",
      "/home/yumizsui/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_sag.py:329: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  warnings.warn(\"The max_iter was reached which means \"\n",
      "[I 2020-08-24 18:10:26,797] Trial 1 finished with value: 0.8515742128935532 and parameters: {'l1_ratio': 0.7603975372415486, 'C': 1955.918787281466}. Best is trial 0 with value: 0.8808095952023988.\n"
     ]
    }
   ],
   "source": [
    "study = optuna.create_study(direction='maximize')\n",
    "study.optimize(objective, timeout=1000,n_trials=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=3.1044375292893234, l1_ratio=0.6575925074474871,\n",
       "                   max_iter=10000, penalty='elasticnet', solver='saga')"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params=study.best_trial.params\n",
    "clf_Log=LogisticRegression(max_iter=10000,penalty=\"elasticnet\",solver='saga', l1_ratio=params[\"l1_ratio\"],C=params[\"C\"])\n",
    "clf_Log.fit(train_feature, train_true)b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train :  0.9450899550224887\n",
      "valid :  0.8808095952023988\n",
      "test  :  0.8703148425787106\n"
     ]
    }
   ],
   "source": [
    "train_pred=clf_Log.predict(train_feature)\n",
    "valid_pred=clf_Log.predict(valid_feature)\n",
    "test_pred=clf_Log.predict(test_feature)\n",
    "\n",
    "print(\"train : \",f1_score(train_pred, train_true, average='micro'))\n",
    "print(\"valid : \",f1_score(valid_pred, valid_true, average='micro'))\n",
    "print(\"test  : \",f1_score(test_pred, test_true, average='micro'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}