Fast.ai/(Fast.ai) Neural Net Tabular data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tabular Data  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using Small MovieLens Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.tabular import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://files.fast.ai/data/examples/adult_sample\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Read CSV in Pandas\n",
    "path = untar_data(URLs.ADULT_SAMPLE)\n",
    "df = pd.read_csv(path/'adult.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education-num</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>native-country</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>49</td>\n",
       "      <td>Private</td>\n",
       "      <td>101320</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Wife</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>1902</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44</td>\n",
       "      <td>Private</td>\n",
       "      <td>236746</td>\n",
       "      <td>Masters</td>\n",
       "      <td>14.0</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>10520</td>\n",
       "      <td>0</td>\n",
       "      <td>45</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>96185</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Unmarried</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>38</td>\n",
       "      <td>Self-emp-inc</td>\n",
       "      <td>112847</td>\n",
       "      <td>Prof-school</td>\n",
       "      <td>15.0</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Asian-Pac-Islander</td>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>42</td>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>82297</td>\n",
       "      <td>7th-8th</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Other-service</td>\n",
       "      <td>Wife</td>\n",
       "      <td>Black</td>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age          workclass  fnlwgt     education  education-num  \\\n",
       "0   49            Private  101320    Assoc-acdm           12.0   \n",
       "1   44            Private  236746       Masters           14.0   \n",
       "2   38            Private   96185       HS-grad            NaN   \n",
       "3   38       Self-emp-inc  112847   Prof-school           15.0   \n",
       "4   42   Self-emp-not-inc   82297       7th-8th            NaN   \n",
       "\n",
       "        marital-status        occupation    relationship                 race  \\\n",
       "0   Married-civ-spouse               NaN            Wife                White   \n",
       "1             Divorced   Exec-managerial   Not-in-family                White   \n",
       "2             Divorced               NaN       Unmarried                Black   \n",
       "3   Married-civ-spouse    Prof-specialty         Husband   Asian-Pac-Islander   \n",
       "4   Married-civ-spouse     Other-service            Wife                Black   \n",
       "\n",
       "       sex  capital-gain  capital-loss  hours-per-week  native-country salary  \n",
       "0   Female             0          1902              40   United-States  >=50k  \n",
       "1     Male         10520             0              45   United-States  >=50k  \n",
       "2   Female             0             0              32   United-States   <50k  \n",
       "3     Male             0             0              40   United-States  >=50k  \n",
       "4   Female             0             0              50   United-States   <50k  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# FillMissing: Look for missing values and deal with them some way.\n",
    "# Categorify: Find categorical variables and turn them into Pandas categories\n",
    "# Normalize : Do a normalization ahead of time which is to take continuous variables and subtract their mean and divide by their standard deviation so they are zero-one variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dep_var = 'salary'\n",
    "cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']\n",
    "cont_names = ['age', 'fnlwgt', 'education-num']\n",
    "procs = [FillMissing, Categorify, Normalize]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set-aside Testset\n",
    "test = TabularList.from_df(df.iloc[800:1000].copy(), path=path, cat_names=cat_names, cont_names=cont_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Databunch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = (TabularList.from_df(df, path=path, cat_names=cat_names, cont_names=cont_names, procs=procs)\n",
    "                           .split_by_idx(list(range(800,1000)))\n",
    "                           .label_from_df(cols=dep_var)\n",
    "                           .add_test(test, label=0)\n",
    "                           .databunch())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>workclass</th>\n",
       "      <th>education</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>education-num_na</th>\n",
       "      <th>age</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education-num</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>Private</td>\n",
       "      <td>11th</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Tech-support</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.9959</td>\n",
       "      <td>-0.0770</td>\n",
       "      <td>-1.2046</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Private</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>Divorced</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0303</td>\n",
       "      <td>-1.0338</td>\n",
       "      <td>-0.0312</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Self-emp-inc</td>\n",
       "      <td>Prof-school</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>False</td>\n",
       "      <td>0.4701</td>\n",
       "      <td>-0.0798</td>\n",
       "      <td>1.9245</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Private</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Prof-specialty</td>\n",
       "      <td>Wife</td>\n",
       "      <td>White</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.7760</td>\n",
       "      <td>0.3310</td>\n",
       "      <td>1.1422</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Private</td>\n",
       "      <td>12th</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Other-service</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>False</td>\n",
       "      <td>-1.5823</td>\n",
       "      <td>0.7718</td>\n",
       "      <td>-0.8135</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Private</td>\n",
       "      <td>10th</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Craft-repair</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.7760</td>\n",
       "      <td>0.1403</td>\n",
       "      <td>-1.5958</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Private</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>Separated</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Not-in-family</td>\n",
       "      <td>White</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.0430</td>\n",
       "      <td>-0.8319</td>\n",
       "      <td>-0.0312</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Self-emp-not-inc</td>\n",
       "      <td>Masters</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Craft-repair</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0303</td>\n",
       "      <td>1.3944</td>\n",
       "      <td>1.5334</td>\n",
       "      <td>&gt;=50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Private</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>Widowed</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Unmarried</td>\n",
       "      <td>White</td>\n",
       "      <td>False</td>\n",
       "      <td>2.2292</td>\n",
       "      <td>-0.2342</td>\n",
       "      <td>-0.4224</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Private</td>\n",
       "      <td>Bachelors</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Adm-clerical</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>False</td>\n",
       "      <td>-1.1425</td>\n",
       "      <td>0.2602</td>\n",
       "      <td>1.1422</td>\n",
       "      <td>&lt;50k</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.show_batch(rows=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We're gonna learn what layers= means either towards the end of class today or the start of class next week, \n",
    "# but this is where we're basically defining our architecture just like when we chose ResNet 34 or whatever for conv nets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Leaner\n",
    "learn = tabular_learner(data, layers=[200,100], metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='87' class='' max='505', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      17.23% [87/505 00:22<01:47 0.9640]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAEGCAYAAABRvCMcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxcZb348c83mUz2fWua7m26pDsNO2WHSwFZvKAsKorIRUW41x+K/kR/ilfvFb14r8pVAQEFEdlcULAgspS1TfclTZuuSdrs+57MPL8/5kw6TSbJ7JPl+3698urMmXPOPKdp5zvPc77P9xFjDEoppVS4xUS7AUoppaYGDThKKaUiQgOOUkqpiNCAo5RSKiI04CillIoIW7QbEAk5OTlmzpw50W6GUkpNGDk5Oaxfv369MeayUJ1zSgScOXPmUFpaGu1mKKXUhCIiOaE8nw6pKaWUiggNOEoppSJCA45SSqmI0ICjlFIqIjTgKKWUiggNOEoppSJCA45SSqmI0ICjlJrSjrd28+KWqmg3Y0qYEhM/lVJqJN/84y7+XlbH7Owk1szOinZzJjXt4SilpqytR5v5e1kdAL9462CUWzP5acBRSk1ZD762j8ykOD57zlxe21NLRV1HtJs0qWnAUUpNSZsON7FhfwOfP38+Xzh/PvG2GB55W3s54aQBRyk15Rhj+NH6cnJT4/nkGXPITonnYyUz+cPWamrbeqLdvElLA45Sasp570AjHx5q4gvnzyfRHgvA59bOY8Dp5LF3D0W5dZOXBhyl1JRijOG/Xi2nID2BG0+bNbh9VnYSly8v4OkPjtLW0x/FFk5eGnCUUlPKm/vq2XK0hTsvXEBCXOxJr91x3nzaewd4+sOjUWrd5KYBRyk1ZRhj+PFr+5iRmcj1a2YOe31ZYTrnLMjhsXcO0TvgiEILJzcNOEqpKePt/Q3sqGrl7ouKsNu8f/zdcd586tp7+ePW6gi3bvLTSgNKqSlj06EmYmOEj6ycPuI+Zy/IZun0NL77lzKe/vAoWcl2spLjyUqO49yFuawtyo1giycX7eEopaaMvTXtzMtJHnbvxpOI8KPrV3JJcT7pSXbqO3p5/0ADv37/CLc+sYl9te0RbPHkoj0cpdSUsbemjVUzM8bcb0lBGj/++KqTtjV29HLxg29x7ws7eP6Os4iNkXA1c9LSHo5SatxyOg23/6aU18tqgz5Xe08/Vc3dLClIC+j47JR4vvWRYrYebeHJ9w+PuN97BxqoqNNekDcacJRS49bBhk5e3VPL9/5ahsNpgjqXeyhs8bTUgM9xzapCzluYywPry6lu6R72+uPvHuKmRz7kK8/vCPg9JjMNOEqpcWtHVQvgCjx/21UT1LnKjrsCzqIgAo6I8L1rlwFw3x92YowrCBpj+Nk/9vOdl/aQnWxnW2ULde1aImcoDThKqXFrR1UrSfZY5uUk87M3KgY/4ANRXtNOaryNwozEoNo0IzOJey5dxBvl9fx5+zGMMfzgb+X86NV9fHR1IY9/5lSMgX9Yyx6oE8IacETkMhEpF5EKEfmal9fvEJGdIrJNRN4RkWKP11aIyPsistvaJ8Ha/qZ1zm3WT144r0EpFT3bq1pYNj2dz58/n7LjbbxZXh/wufbWtLFoWioiwd/sv+WsOaycmcF3XtrD117YyS/eOsDNp8/iR9evZHlhOoUZiby2J/j7TpNN2AKOiMQCDwHrgGLgRs+AYnnaGLPcGLMKeAB40DrWBjwF3GGMWQqcD3gWN7rZGLPK+tGvEWrKefrDo7xb0RDtZoRVv8PJnmNtrJiRzjWrCynMSAy4l2OMYW9NO4sLAh9O8xQbI/zgn5fT1t3P70sr+Zfz5vHv1ywjJkYQES4pzuediga6+gZC8n6TRTh7OKcBFcaYg8aYPuAZ4GrPHYwxbR5PkwH3v6RLgR3GmO3Wfo3GGK0zoRSubKv7/riTTz++MSTZW+NVeU07vQNOVs7MIC42hn85bx6bjzTz4aEmv891rLWH9p4BFk0LLEPNm8XT0vjR9Sv5/rXL+dpli0/qOV1SnE/vgJMN+yf3lwJ/hTPgFAKVHs+rrG0nEZEvisgBXD2cu6zNCwEjIutFZIuIfHXIYY9bw2nflBH6xyJyu4iUikhpfX3g3XClxputR1twGshIsvP5p7bwZvnk7OTvqGoFYOUM17yZj5XMJCclnofeqPD7XOU1ru+2S4JIGPDmmtWF3HT6rGHDdKfNzSI1wabDakOEM+B4CwTD+sLGmIeMMfOBe4H7rM024BzgZuvPa0XkIuu1m40xy4G11s8nvb25MeZhY0yJMaYkN1dLUajJY9NhV3mWP33xbBZOS+H2JzezYf/k+1K1vbKFzKQ4Zma5bvInxMVy29q5bNjfMJi95it3htrCEAeckcTFxnDBojz+sbcu6HTuoYwx7DnWNvaO41A4A04V4FmOdQZwbJT9nwGu8Tj2LWNMgzGmC3gZOAXAGFNt/dkOPI1r6E6pKWPT4SaKC9KYnpHIk7eezvzcFG77dSnvTbJ7OturWlg+I+Ok3sPNp88iLcHmdy+nvKadwoxE0hLiQt3MEV1SnE9TZx9bjjaH9Lzrd9dy+U828PcJ2HsKZ8DZBBSJyFwRsQM3AH/23EFEijyeXgHstx6vB1aISJKVQHAesEdEbCKSYx0bB1wJ7ArjNSg1rvQNONlW2cKpc7IAyEy289RnT2NOdjKf/XUp2yr9++Y/XnX3Odhf18HKGeknbU9NiOPTZ89l/e5a9vtR02xvTVtQEz4Dcd6iXOJiJeTDaq/uds1HevjtgyE9bySELeAYYwaAO3EFjzLgWWPMbhG5X0Susna700p73gZ8GbjFOrYZV8baJmAbsMUY81cgHlgvIjus7dXAI+G6BqXGm13HWunpd3LqnMzBbdkp8fz2c6eTHB8b0P2N8Wj3sVYcTsOKGcPrnt1y5mwA1u/2bSJo74CDg/WdIctQ81VaQhxnzMvmtT21Qc0f8uRwGt4oryM13sbGw00T7gtGWOfhGGNeNsYsNMbMN8Z8z9r2LWPMn63HdxtjllrpzRcYY3Z7HPuU9doyY8xXrW2dxpg1xpgV1mt3a/aamkpKD7sytEqsHo5bTko8/3zKDN7YW0dDR280mhZS2wcTBtKHvZadEs+c7CR2Vrf6dK4DdZ0MOE1IM9R8dUlxPocaOjlQ3xmS82092kxzVz/3XbmE1AQbj2yYWL0crTSg1ASy6XAzc3OSyU2NH/badWtmMOA0k2LhsB1VLUxLSyAvLcHr68tnZLCr2rcb5+W14clQ88XFS/IBQjas9vreOmwxwrrlBdx02ixe2XmcyqaukJw7EjTgKDVBOJ2G0sNNlMzO9Pp6UX4qK2dm8PzmqpAN4UTLjqpWVnjp3bgtL0yjuqWbRh96c3uPt2OPjWFOTnIom+iT6RmJLCtM47U9wdWBc3u9rJbT5maRlhDHp8+eQ4wIj717KCTnjgQNOEpNEAcbOmju6h9MGPDmujUz2FvTzu4JmjYL0Nrdz6GGTlaOsm7NskJXMPJlWG1vTTsL8lKIi43Ox93FS/LZWtlCfXtwQ52VTV3sq+3gIqvXVJCeyFUrp/P7TZW0dvWPcfT4oAFHqQli02FXem3JHO89HICrVkzHbovh+c1VkWpWyO207t+M1sNxB5xdPgWctognDHi6pDgfYwi6KoT7+IsWnygfedvaeXT1OfjtxiNBnTtSNOAoNUFsOtREToqduaMMDaUnxXFpcT5/3FZN78DEzKfZbk3qXFE4cg8nLSGOuTnJY/Zwmjv7qG3rjXhKtKfigjSK8lJ48LV9QS1Z8PreOublJp80NFg8PY1zFuTwxLuH6RtwhqK5YaUBR6kJYtORJkpmZ41Z7fj6kpm0dPVP2PL4O6pamJOdRHrS6JM0lxWmj5k4sLfGveha5DPU3ESEn9y4mraefu58eiv9Dv8DQ0fvAB8ebBpMQvD0uXPnUdfey5+3jzavfnzQgKPUBFDT2kNlU/eow2lu5yzIYVpaAs9FaVjt0Q0Huf4X7wVcbseVMDBy78bNl8SBvVYNtWj2cACWFKTxHx9dzsZDTTzwt71+H//O/nr6HE4uXDx8NZZzi3JYlJ/KoxsOjvtkEQ04Sk0Am6z5N6fNHTlhwC02RvjoKYW8ta+eurbIrzr54pZqNh1u5pO/2sgnf/WhX3W/6tp6ON7aM+r9GzdfEgfKa9rJSrZ7TSOPtGtXz+BTZ87mkQ2H+OuO4ye9Zozhnf0NfP6pzbx/oHHYsX8vqyMtweY1Q1FEuG3tXPbWtPP2OK9OrQFHqXHkWEs3Vc3D51WUHm4iyR5LcYFvQ0PXrZmBw2n4Q4Tn5PT0Oyivbedza+dy3xVL2FHVyhU/3cCXn91GTevYwW9wwucoGWpuviQOlNW0syg/NIuuhcJ9VxSzelYGX3l+OxV1ruG+TYebuOHhD/jErz5k/e4aPvPExpPq4jmdhjf21nH+ojxsI2TaXbVqOnmp8Tw6zieCasBRahy594UdrPvvDXxw8ORvuZsON7N6VsaIHzhDzctNYc3szIjPydl9rA2H01AyJ4vb1s7j7a9cwO3nzuMvO45zx1Obxzx+R1ULMQJLp48dWMdKHHA6DftCuOhaKNhtMfzvzaeQZI/l9ic386nHNnL9L97nQH0n3/5IMe/ceyGzs5K59debBhfY21bVQmNnHxctGXlx43hbLLecNYcN+xsoOz5+U+I14Cg1jhxv7aG9d4BPPbZxsFZYW08/ZTVto86/8ea6NTPYX9cR8mrFo3EvG7DK6qGkJ8Xx9XVLuOfShWyrbBlzVvz2qlYW5qeSZLf59H6jJQ4cbeqiu98R9fs3QxWkJ/LTG0/hSGMXO6ta+Pq6xWz46gV8+uy5TM9I5LefO53ZWcl81go6/yirIzZGOH/hyAEHXJW0E+NieXTD+J0IqgFHqXGksaOXK1cUUFyQxuef2syzmyrZcqQZY/A74Fy5ooCcFDv3PLcjYhMDt1e2kJ8WT/6QkjTrlhUA8PLO494OA1xFNrcdbR4MVr4YLXHg3QOuHsLS6WPfD4q0M+dn8+q/ncuGey/kX86bT6I9dvC1nJR4nraCzq1PbOK5zZWUzM4cM2svI8nOx0+dyZ+3V1MbhXt3vtCAo9Q4MeBw0tzVz4K8FH572+mcU5TLV1/YwfdfLiM2Rvz6IAZXKf9ffGINVc1dfOmZrSFfCMybkTLMZmYlsWJGOi/vGrnEy+tldbT1DLBueYHP77fcmqszdFjNGMMT7x5m6fQ0n4bnomF+bgop8d57ctlW0JmTnUxtW++ow2mebj17Lg6n4Yn3DoewpaGjAUepcaKpqw9wfdgkx9t49FMlXLVyOvtqO1g2PY3kET6cRlMyJ4v7r17G2/vq+c9Xynw+7oODjVz4ozf96hm1dvdzsKHTa4VncPVytle2eE2KAHi2tJKC9ATOWZDj83suLXQFk6GJA+9UNLC/roNbz547bhIG/OUOOl+6cAEfL5nl0zGzspO4bNk0fvvBETp7B8LcQv9pwFFqnGjscAWcnGQ74LrB/N8fX8X/vXwx/3rJwoDPe+NpswbTcV/wcW7O33bVcLChkw8PDU/RHYn7Q3+kOTSXL582eO6hjrd28/a+eq5bM4PYGN8DxEiJA4+/e5iclHiuXOl7b2k8yk6J5/9cumjM4TRPt62dR1vPAM+WVoaxZYHRgKPUOOEOOFlWwAGIiRFuP3c+FyzybUhlJN+8spgz52Xz9T/s9GnRLve8n81HfE84GCxJM0IPZ3Z2Mkunp3m9j/PilmqcxpXo4K+hiQMH6zv4x946PnHGLOJtsaMcOTmdMiuTNbMzeezdQwwEUNUgnDTgKDVONHa6bnxnp4R+kmJcbAwP3XwKeanx/MuTpaMu0tbROzCYWlvqR8DZUdnK7OwkMpLsI+5z+fICthxt4VhL9+A2YwzPllZyxrwsZmf7v4TAisL0kxIHfv3eYeyxMdx8+my/zzVZfG7tPCqbulm/O7TLWwdLA45S48TgkFrKyB/YwchKtvOLT6yhtq2Xl0apu7XtaAtO4yoHs7Oq1ecioDuqWsYsSbNu2fBhtY2HmjjS2MXHSmb69D5DeVYcaO3u57nNVVy5smBcVBeIlkuK85mdncQj46zcjQYcpcaJxs5ebDFCWoLv4/X+WlaYztycZN7eN3Kds02Hm4gR172APofTpyUA6tp7ONbaM2LCgNu83BQWT0vllV0nhtWeLa0iJd42mDrtL8/EgedKK+nqc3Dr2XMDOtdkERsj3HbOXKqau6gLch2eUNKAo9Q40djRR1aynRg/bpoH4tyiHD442DRiz2XzkWYWT0vjvIW5AJQeHntYbUel7yVpLl9eQOmRZmrbemjv6eflncf5yMrpJ81F8Yc7cWBbZStPvHeY0+ZkDfZ6prKPnTqTd+69cNicqGjSgKPUONHQ0ReW+zdDrS3KpbvfwWYvgWTA4WTL0WZK5mSSmxrP7OwknxIH/ClJc/nyaRjjGlb7647jdPc7+FiJ/8kCnpYXpvOPvbVUNXfzmbPnBHWuySLeFktC3PhKmtCAo9Q40djZG7b7N57OnJ9NXKzwlpflA/bWtNPV52CNVZV4zexMNh9pHvM+gD8laRbkpbIwP4WXdx7n2dJKivJS/J7UOtTywnScBgozErmkePiaMWp8CGvAEZHLRKRcRCpE5GteXr9DRHaKyDYReUdEij1eWyEi74vIbmufBGv7Gut5hYj8RCbqrC6lhnAPqYVbcryNNbMzeXvf8FL2pVY6tLuMTsnsLBo7+zjcOHINNGOMlTDg+zDWumUFbDzcxJajLXz81JlBT850v/ctZ832ucCpiryw/WZEJBZ4CFgHFAM3egYUy9PGmOXGmFXAA8CD1rE24CngDmPMUuB8wD3l+efA7UCR9XNZuK5BqUhq7OglOzkymVXnLsyl7HjbsCWPNx1pZnp6AtMzEgEGezqjDatVNXfT3NXv06JpbpcvL8AYsMUI16wuDOAKTnba3Cz+9+ZT+PRZUztZYLwL51eB04AKY8xBY0wf8AxwtecOxhjPMq/JgLvffimwwxiz3dqv0RjjEJECIM0Y875x9fF/A1wTxmtQKiJ6+h109jnIjsCQGsC5Ra6EgA0evRxjDKWHmyjxKBJalJdCWoKNzUeaRjyXe8LnSj8CzsL8FJZOT2Pd8gJyQnDfSkS4fHkBdpv2bsazcP52CgHP2gpV1raTiMgXReQArh7OXdbmhYARkfUiskVEvupxTs/aHF7PaZ33dhEpFZHS+vrAlrpVaiw9/Q7+sTf4yXWNneGdgzNUcUEa2cn2k5aBrmruprat96RlrGNihFNmZ46aqba9sgV7bAyL/FgGQER4/o6z+K/rVwZ2AWpCCmfA8TYoO+zOozHmIWPMfOBe4D5rsw04B7jZ+vNaEbnI13Na533YGFNijCnJzc0NpP1Kjen3myq59YlSn+aqjMY9Sz5SQ2oxMcLaohw27G/AaVWRdg+brRmyjPGaWZnsr+ugxSouOtT2qlaKp6f53btItMdqj2SKCedvuwrwnDo8Axh5erNryM09PFYFvGWMaTDGdAEvA6dY2z3zJ8c6p1JhtdVa3OydiuDWkndXGYjUkBq47uM0dvaxxypjs+lwEynxNhZPOzm1eY3V49l6dHgNNofTsKu6dcwJn0pBeAPOJqBIROaKiB24Afiz5w4iUuTx9Apgv/V4PbBCRJKsBILzgD3GmONAu4icYWWnfQr4UxivQalRba9y9WzeDTLguGubheJ+hq/OKXItA/CWVXVg8xHXMtZDqzWvmunaVurlPs6B+g66+hx+JQyoqStsAccYMwDciSt4lAHPGmN2i8j9InKVtdudVtrzNuDLwC3Wsc24MtY2AduALcaYv1rHfB54FKgADgCvhOsalBpNa1c/hxo6SYyLZeOhJnr6fas55o37Hk4k0qLd8lITWFKQxtv76mnt7qe8tt3rqqJJdhtLp6d5vY+z3ao8vXKm9nDU2Pxf0ckPxpiXcQ2HeW77lsfju0c59ilcqdFDt5cCy0LYTKUCsqPa9WF70+mz+NU7h9h8pJmz/Vg8zFNjRy8JcTEkBVjeJVDnLszhsXcO8c7+BoyBkiH3b9xOmZXJM5uO0u9wEmfNc6lr7+Hhtw+SmRTHvJyUSDZbTVB6x06pALm/3X9u7TxsMRLUfZzGjj6yk+MjvjrleUW59DsMP3ujwrWM9SzvQ2MlczLp6Xey55jrfs+xlm4+/ssPqG7p5qGbTgl7/Tc1OYS1h6PUZLatspV5uclMS0/glFmZQd3Haejsi1hKtKc1czJJjIul7HgbK2akj1iaxp25VnqkmcwkOzc+8gFt3f08+dnTWDN7+DCcUt5oD0epABhj2FbZwirrZvnZC3LYWd1Kc6f31OGxNHX2RqRw51DxtljOmOcKGEPToT0VpCdSmJHIX3cc4/pfvkdn3wBPf+4MDTbKLxpwlArA8dYeGjp6B8vxn1OUjTHw/sHGgM7nGlKLfA8HXOnRgNeEAU9rZmey5WgLDic8c/sZLNdUaOUnHVJTKgA73OVcrICzYkYGKfE23qlo4PLl/i0kZoxxBZwo9HAArl1dyLGWbs5fNPoE6WtPKeRIUxcPfmwl83M1SUD5TwOOUgHYVtlKXKywpMBVziUuNoYz5mXzzn7/7+O09w7Q53BGrYeTkWTnG1cMras73AWL8rhgUV4EWqQmKx1SUyoA2ytbWFKQRrztRBrzOQuyOdrUxdFRSvl7E40qA0pFgwYcpfzkcBp2VrcOq47snrn/7gH/ejmDddSiNKSmVKRowFHKTwfrO+joHRi8f+M2PzeF/LR4v4fVGtw9nCgNqSkVKRpwlPKTu37aqiHlXESEcxbk8u6BExWYfdHYGfk6akpFgwYcpfy0vbKFlHib13Iu5xRl09LVP1iB2RdNHZGvo6ZUNGjAUcpP26taWDEj3Ws5l7Pnu+7j+FPmprGzj7QEm64NoyY9/ReulB96+h1WGRjvNcfy0hJYmJ/i132cho7oVBlQKtI04Cjlh7LjbfQ7zLD7N57OWZDLxsNNdPf5tlxBNKsMKBVJGnCU8sMOK2FgaIaap/MX5dI34OQ9H9OjGzt7dQ6OmhI04Cjlh+2VLeSlxjMtLWHEfU6fl0WyPZa/l9X6dM5olrVRKpI04Cjlh21VLaycmTHqujXxtljOW5TL62V1Y6ZHO5yGpq4+cnRITU0BGnCU8lFrdz8H6ztZ6UOV5IuX5FPX3svO6tZR92vu6sMYrTKgpgYNOEr5aLcVPEbKUPN0waI8YgReH2NYralT66ipqUMDjlI+OtLkKso5Lzd5zH0zk+2UzM7itbK6UfdrsOqo6aRPNRVowFHKR9XN3cTGyKgJA54uWpJH2fE2qlu6R9zHXSlay9qoqUADjlI+qm7pZlpaArZY3/7bXFycD4w+rDZYKVp7OGoK0ICjlI+qmrsozEz0ef/5uSnMzUnm76MMqzV29hEjrkXQlJrswhpwROQyESkXkQoR+ZqX1+8QkZ0isk1E3hGRYmv7HBHptrZvE5FfeBzzpnVO92u6BKGKiOrmbmZk+B5wAC5ekscHBxrp6B3w+npDRx9ZyXZivdRlU2qyCVvAEZFY4CFgHVAM3OgOKB6eNsYsN8asAh4AHvR47YAxZpX1c8eQ4272eG30u7JKhUC/w0lNWw8z/OjhgCs9us/hZMO+eq+vN3b0kp2s92/U1BDOHs5pQIUx5qAxpg94BrjacwdjjGcN92TA90VElIqgmtYenAa/htQA1szOJD0xjtdGuI/T2NmnKdFqyghnwCkEKj2eV1nbTiIiXxSRA7h6OHd5vDRXRLaKyFsisnbIYY9bw2nflBGmfIvI7SJSKiKl9fXev10q5auqZlemWWFGkl/H2WJjuHBxHm/srcPhpepAU6eWtVFTRzgDjrdAMOx/nDHmIWPMfOBe4D5r83FgljFmNfBl4GkRSbNeu9kYsxxYa/180tubG2MeNsaUGGNKcnNzg7wUNdW5U5v9HVIDV3p0c1c/W442D3utoaNXM9TUlBHOgFMFzPR4PgM4Nsr+zwDXABhjeo0xjdbjzcABYKH1vNr6sx14GtfQnVJhVW31cAoyfJuD4+nchbnExcqwYp69Aw7aewY04KgpI5wBZxNQJCJzRcQO3AD82XMHESnyeHoFsN/anmslHSAi84Ai4KCI2EQkx9oeB1wJ7ArjNSgFuFKi81LjibfF+n1sWkIcp8/N5m+7augdOLFGzomyNjqkpqaGsAUcY8wAcCewHigDnjXG7BaR+0XkKmu3O0Vkt4hswzV0dou1/Vxgh4hsB54H7jDGNAHxwHoR2QFsA6qBR8J1DUq5Vbd0+50w4OkzZ8/hSGMX3/jDLoxxjSy7qwxo0oCaKmzhPLkx5mXg5SHbvuXx+O4RjnsBeMHL9k5gTYibqdSYqlu6fSraOZKLluRz90VF/M/r+1k8LZXb1s4brKOWowFHTRFhDThKTQZOp+FYSzfrlhUEdZ67LypiX20733+5jPl5KTS5ezg6D0dNEVraRqkx1LX30u8wQQ2pAcTECP/1sZUsmpbGXU9vZeOhJkCH1NTU4VPAEZH5IhJvPT5fRO4SkcDHF5SaQKpbXMsS+FvWxpsku41HPrUGuy2G35dWYo+NISVeBxrU1OBrD+cFwCEiC4BfAXNxpSQrNem5J30GMgfHmxmZSfzyk2uIixWyku2jLlet1GTi61crpzFmQESuBf7bGPNTEdkazoYpNV4MVhkIUcABKJmTxf/evGZweQKlpgJfA06/iNyIK235I9a2uPA0Sanxpbqlm8ykOJLsoR36usRaL0epqcLXIbXPAGcC3zPGHBKRucBT4WuWUuNHdXNwc3CUUi4+fWUzxuzBKqwpIplAqjHmP8PZMKXGi+qWbhbkpkS7GUpNeL5mqb0pImkikgVsx1Wt+cGxjlNqojPG+L3Sp1LKO1+H1NKttWs+CjxujFkDXBy+Zik1PjR19tHT76QwBCnRSk11vgYcm4gUAB8D/hLG9ig1rgSzLIFS6mS+Bpz7cRXhPGCM2WRVcN4fvmapSPvWn3bxys7jITlXeU07tzy2cbBW2EQWjpRopaYqnwKOMeY5Y8wKY7tqBXoAACAASURBVMznrecHjTH/HN6mqUgxxvDMxkqeLa0ce+cxdPYO8PnfbuatffX8cWt1CFoXXe51cGb4udKnUmo4X5MGZojIH0SkTkRqReQFEZkR7sapyOjqc9DncLKzum2wdH6gvvmnXRxu6CQ3NZ6XQ9Rjiqbqlm5S4m2kJWr5GaWC5euQ2uO4Fk+bDhQCL1nb1CTQ3OWqWtzQ0Utde+DDYM9vruLFLdXcdVERnz5rDluOtnDMugcyUVU1dzMjM1HLzygVAr4GnFxjzOPGmAHr5wkgN4ztUhHU0tU/+HhnVWtA59hf2843/7iLM+dl86ULi7h8uauUf6R6OX0DTr7+4g5++daBkJ63qrlLM9SUChFfA06DiHxCRGKtn08AjeFsmIocdw8HYGe1/wGnu8/BnU9vJckey//csIrYGGFuTjLFBWn8NQIBp9/h5Eu/28LvNlbyH6/sDVnyAwS/0qdS6gRfA86tuFKia4DjwHW4yt2oSaDZ6uHYbTHsPuZ/wLn/L7spr23nxx9fRV5awuD2K1YUsPVoy2BqcTgMOJz86++3sX53LfddsYRVMzP4yvM7OFDfEfS5W7v7ae8Z0B6OUiHia5baUWPMVcaYXGNMnjHmGlyTQNUk0GL1cE6fm+V3D+e3Hx7hdxsr+cL58zl34cmjrFdYw2qh7HF4cjgN/+e57fx1x3Huu2IJt62dx88/cQrxthjueHIznb0DQZ1/MEMtUzPUlAqFYFb8/HLIWqGiqrnT1cM5Z0EOtW291LX3+HTc62W1fPOPu7hwcR5fvmThsNfn5CSzdHp4htWcTsNXn9/Bn7Yd46uXLeK2tfMAKEhP5Kc3ruZAfQf3vrAjqKw7d89Mh9SUCo1gAo6m7UwSzV19pMTbWD0rE4Dd1W1jHrOjqoU7n97K0unp/PTG1dhivf9Tuny5a1itqrkrpG3+wfq9vLClin+7eCFfOH/BSa+dtSCHr/zTYv6y4ziPvXs44PeottqsQ2pKhUYwASe4CRuTyIH6Dnr6HdFuRsBauvrISIqjeHoaImMnDlQ2dXHrE6VkJdv51adLSB5lieQTw2o1IWtvV98AT75/hKtXTeeuixZ43eeO8+bxT0vz+Y+Xy9h8pCmg96lu6SbeFkNOij2Y5iqlLKMGHBFpF5E2Lz/tuObkjEpELhORchGpEJGveXn9DhHZKSLbROQdESm2ts8RkW5r+zYR+YXHMWusYypE5CcS5QkSXX0DrPufDdz4yAe09fSPfcA41NzVT2aSnZR4G3NzkkcNOC1dfXz68Y30O5z8+tZTyUtNGHFfCM+w2vrdNXT1ObjptFkjzo8REX54/UrSE+P4zftHAnqfKmsdHJ2Do1RojBpwjDGpxpg0Lz+pxphRp16LSCzwELAOKAZudAcUD08bY5YbY1YBDwCeSx4cMMassn7u8Nj+c+B2oMj6ucynKw2T/bUd9A042Xq0hU8++iGtXRMv6Lh7OADLC9PZPULAGXA4uf3JzVQ2dfPwJ9ewIC/Vp/NfsaKAbZWhG1Z7cUs1hRmJnDona9T90hLiWD0rg7LjYw8RelPd0q3DaUqFUDBDamM5Daiw6q71Ac8AV3vuYC154JbMGMN0VsXqNGPM+8Z1N/g3wDWhbbZ/ymvaAbjviiWUHW/n5l99QHNn3xhHjS/uHg7AsunpHGvtodFL4c03y+vZeKiJ716zlNPnZft8/lAOq9W19fBuRQPXri4kJmbsnseSgjQO1HcGNORZbVUZUEqFRjgDTiHgWQ2yytp2EhH5oogcwNXDucvjpbkislVE3hKRtR7nrBrrnNZ5bxeRUhEpra+vD+Y6RrW3pp2EuBg+c/ZcfvmpNeyr7eCmRz/0+oE9XjV39ZFp9XCWFaYD3u/jPLOpkpyUeD56in9l9GZnJ7OsMI2/hGBY7U/bjuE0cO0pXn/twxQXpOFwGvbVtvv1Pl19AzR29mlKtFIhFM6A4+3r57AejDHmIWPMfOBe4D5r83FgljFmNa7066dFJM3Xc1rnfdgYU2KMKcnNDV8VnvLaNhbmpxIbI1ywKI9HP1XCwfoObnxkYvR0BhxO2nsGyLB6OEsL0wDYfezkYai6th7eKK/jujUziBshI200ly8vYHtlC3VtvqVcj+TFrdWsnJnBfB+XfC6e7rqePcf8G1arqHNNHNUhNaVCJ5wBpwqY6fF8BnBslP2fwRoeM8b0GmMarcebgQPAQuucnl+vxzpn2JXXtLMo/8S9jHMX5vKrW05lX20HL06A8vwt3a57Tu4eTlpCHHOyk4bVVHt+SxUOp+Hjp84cdg5fuO+37AqgkoFb2fE2yo638dHVvvVuAGZmJpESb/PrPo7DafjOS3tIS7Bx9oKcQJqqlPIinAFnE1AkInNFxA7cgKvi9CARKfJ4egXWom4ikmslHWAt9lYEHDTGHAfaReQMKzvtU8CfwngNo2ro6KWho49F006+eX5OUQ7ZyXb21fg3jBMN7ioDmcknUn+XFaafFBiMMfx+UyWnz81ibk5yQO+z2Po78ren4ekPW6uxxQgfWTlmguSgmBhh8bRU9vgRcB5/9xCbjzTznauXkpsaH0hTlVJehC3gGGMGgDtxrRRaBjxrjNktIveLyFXWbneKyG4R2YZr6OwWa/u5wA4R2Q48D9xhjHFPpvg88ChQgavn80q4rmEs7oAyNOAALMxPpdzP+wbR4K6j5h5SA1fAqWruHhwS/OBgE0cau7jhtMB6NwCpCXHMzk7y64Pfk8Np+NO2as5flEtWsn/zYoqnp1F2vB2nc+ypYxV1HfxwfTmXFOdzzSrfe1JKqbGFdVUpY8zLwMtDtn3L4/HdIxz3AvDCCK+VAstC2MyA7R0l4CyalspzpZUYY8b1PA53UHEPqYErNRpcw19ri3L5/aajpCbYWLesIKj3Ki5wffAH4r0DDdS29fL/PuL/un/FBWn8pvcIlc1dzM4euYfmcBrueW47ifZYvnftsnH9e1NqIgrnkNqkV17TTlayndyU4cMuC/NT6exzhLVSciicuIdzotew1LrRvqu6jdaufl7eVcO1qwtJiIsN6r2WFKRxuLGTjgCKar64pZrUBBsXLs7z+1h34sBY93Ee2XCQbZUtfOeqpWNOaFVK+U8DThD21roSBrx9E140zZVF5W86bqS57+FkePRwMpLszMxKZFd1K3/cVk3fgDPgZAFPxQVpGAPlNf4Nq3X2DvC3XTVcuaIgoKDnziIc7f7R/tp2Hnx1H5ctncZVftwjUkr5TgNOgJxOw/7adq/DacDgLPzymuDXZQmn5q5+bDFCypB6aMsL09lZ3crvNh5leWE6S6enB/1egaYor99dQ3e/g2tX+z+cBpAQF8u8nOQR7x+5h9JSEmz8uw6lKRU2GnACVNXcTVefY8SAk54YR0F6woTo4WQk2Yd9yC6dns7Rpi721rSHpHcDUJCeQHpinN+JAy9tP0ZhRiIlszMDfu/i6WkjBroPDzayvaqVr69bTI6X4VGlVGhowAnQXmtYaKSAA1am2jhPjW7u7D8pYcDNnTiQEBfDVatCM8QkIhQXpLHHj8SBrr4B3j3QyKVL830qZTOS4oI0jrX2DA4henppxzGS7bFcuUKH0pQKJw04AXIHkoX5IwecRdNSqajvwOFDOm60uMraDE8zdpe4uXx5AWkJwwNSoIqnp7H3eBsDDqdP+2/Y30DfgJNLluQH/b7AsN5Vv8PJK7tquLg4n0R7cEkRSqnRacAJUHltOzOzEofd+/C0MD+VvgEnRxo7I9gy/7R09Z+UMOCWlWzn4U+u4evrloT0/YoL0ugdcHLYx7+T18tqSU2wcerc0StDj2VJgff7R+9UNNDS1c9HtHejVNhpwAmQq6RN2qj7LMwf/5lqI/VwAC5dOi3kM+3dH/xDa7V543Qa/rG3jvMW5gZUv81TTko8eanxw3o4L20/RlqCjbULtYSNUuGmAScAvQMODjZ0DqY+j2RBXgoi4zdTzRjj6uEkh27IbCwL8lKIixWfJoBuq2qhoaOPS4qDG05zG5o40NPv4NXdtVy2bBrxNh1OUyrcNOAE4EBdJw6nYdG00Xs4SXYbs7KS2Fc3Pns4XX0O+hzOEXs44WC3xVCU51tts9fLaomNEc5f6P9kT2+KC9I4UO9aMA9c6/t09A5osoBSEaIBJwDlta4Py8WjZKi5LcxP9auI53sHGjj/h2/QHoHlqpu7hpe1iYTRUpQ9vV5WR8nsTNJD1L7i6Wn0Owz7rS8Af9lxjKxkO2fN930xOaVU4DTgBKC8poO4WPGpcvKi/FQONXTSO+DbipPPlVZxuLGLqubwl8Rp8VK4MxKWFKTR0NFLXfvIa+NUWnOALg4yO81TsUfiQFffAK+X1XH58mnYgrw/pJTyjf5PC0B5TRvzc1N8upFdlJ/CgNNwqGHsrKwBh5M3yusAIrJ4m7uHk5EY4R5Ogbu22cg9v9fLagG4OET3b8C18mhiXCx7jrfx97I6uvsdOpymVARpwAlAeU27T8NpcGJi6L7asRMHthxtGex1NHmZoBhq7qUJMv0s9x+s4hFSlD29vreOebnJAa+/401sjLC4IJWy4228tP0Y+WnxgwvDKaXCTwOOn1q7+znW2sNCHwPOvJwUbDHi030c97d6gKYI9HC8Fe6MhPSkOAozEkdMHGjv6eeDg40hHU5zKy5IY1d1G2+V13PF8unEBlG9QCnlHw04ftpvzanxtYdjt8UwNyfZp8XYXiur5cx5rhvYkQg4zZ3WPZzEyPZwwHUfZ88Iy02/va+BfocJT8CZnkZH7wB9DicfWRnc+j5KKf9owPHTiUXXRk+J9rQwP3XMyZ+HGjo5WN/JPy3NJy3BFrF7OCnxNuy2yP8zKJ6exqGGTrr7hidTvF5WS0ZSHKfMygj9+1rDeTMyE1k1M/TnV0qNTAOOn8pr2klNsDE93fcFuhbmp3K0qcvrh6ubezjtoiX5ZCXbaeoKf1q0q1J0ZIfT3IoL0nAahvX8HE7DG+V1XLAoLyzZY4umpZIYF8s1qwp1GQKlIkwDjp9cJW28L7o2kkXTUjAGKupGThx4bU8ti6elMjMricxke4R6OP0RnfTpaaTEgS1Hm2nu6ueiJaGZ7DlUkt3Gq/92LnddVBSW8yulRqYBxw/GGMpr231OGHBzV5Qe6T5Oa1c/pUeaBz9ks5LsEUsaiFYPZ0ZmIqnxNvYcd93H6Xc4+ePWar7+4k7iYoVzF+aG7b1nZiVFZRhRqalu5FLHapj6jl5au/tZmDd6DbWhZmcnY7fFjHgf5819dTicJ26SZybb/V6kLBDNXf3Mzg5d2rE/YmKEJQVpbK9s5fF3D/HohkNUt3RTlJfCQzedEtIlEZRS44MGHD/st+bSjLYGjjexMcKC3JQRF2N7bU8tOSnxrJzhuomdlezq4RhjwnqfwVUpOnof7EsKUvn1+0fYWd1KyexMvnPVUi5cnBfUQmtKqfFLA44f3CnRC/L96+GA62b1hwcbh23vdzh5a18965ZNG/ygzUyy0zvgpLvfQZI9PL+iAYeT9p6BiJe18fSxU2fS0+/k+pIZlOgETKUmvbAOZIvIZSJSLiIVIvI1L6/fISI7RWSbiLwjIsVDXp8lIh0ico/HtsMex5SGs/1D7a/rID0xjtwA1r1fmJ/KsdYe2oYU5dx0qIn2noGT5pxkWzP/w3kfp6XbqjIQxR7O0unp/OC6FRpslJoiwhZwRCQWeAhYBxQDNw4NKMDTxpjlxphVwAPAg0Ne/zHwipfTX2CMWWWMKQl1u0ezv7aDhfkpAQ1zudfOefydwydVgn6trBa7LYZzik4sAOYuNeOemBkO7ioDkS5ro5SausLZwzkNqDDGHDTG9AHPAFd77mCM8bwzngwY9xMRuQY4COwOYxt9ZoxhX107C/L8u3/jdvrcbE6dk8mP/76P07//Ol9/cSe7j7XyelkdZ8/PPmnoLMtaEC2c9dSao1QpWik1dYXzHk4hUOnxvAo4fehOIvJF4MuAHbjQ2pYM3AtcAtwz5BADvCoiBvilMeZhb28uIrcDtwPMmjUrqAsBaOjoo6WrnyI/M9TckuNtPHfHWeyoauGpD47w4pYqfrfxKAD/ct68k/Z1z40J51wc97mjOaSmlJpawtnD8TbuZIZtMOYhY8x8XAHmPmvzd4AfG2O8zZQ82xhzCq6hui+KyLne3twY87AxpsQYU5KbG/ycDveiXf5mqA21YkYGD1y3ko3/92K+eWUxly2dxuXLTq7plRXRezjaw1FKRUY4ezhVwEyP5zOAY6Ps/wzwc+vx6cB1IvIAkAE4RaTHGPMzY8wxAGNMnYj8AdfQ3dshb/0Q7ioBRQFkqHmTnhTHZ8+Zy2fPmTvstbSEOGLkxHo14RCtStFKqakrnAFnE1AkInOBauAG4CbPHUSkyBiz33p6BbAfwBiz1mOfbwMdxpifWUNtMcaYduvxpcD9YbyGQftqXTXU8lL9z1DzV0yMkBnmagPNXf3YYoSUeM2MV0pFRtg+bYwxAyJyJ7AeiAUeM8bsFpH7gVJjzJ+BO0XkYqAfaAZuGeO0+cAfrCwxG64st7+F6xo8uTLU/KuhFozMZHvYezgZSXYtYKmUipiwfr01xrwMvDxk27c8Ht/twzm+7fH4ILAyhE30WUVdB5eEcLnjsYS7nlpzZ78mDCilIkorGPqgsaOXxs4+FgSYoRaIzOS4sM7DcZW10YQBpVTkaMDxwf7BhIHgMtT84VoTJ5xDav2aMKCUiigNOD5wB5yFIcpQ80VmkmtNHGOGZZKHhPZwlFKRpgHHB/tr20mJtzEtzfdVPoOVlWxnwGlo7x0I+bmNMa4eTrL2cJRSkaMBxwf7aztYkBdYDbVAhbPaQFefgz6HU3s4SqmI0oDjg/11HREdToPwVhtwp1trlppSKpI04IyhubOPho5eigIs2hmowYrRYUgcaNHCnUqpKNCAMwZ3wkAgi64FIyvJ3cMJfWr0iR6OBhylVORowBlDqIp2+ivTuqEfjns47qUJdEhNKRVJGnDGsL+2g2R7LNPTI5ehBpASbyMuVsIyF+dE4U7t4SilIkcDzhj217VHPEMNQEQG5+KEmruCgU78VEpFkgacMeyv7YhohQFPWcnhqafW3NVHaryNuFj99SulIkc/cUbR2tVPXXtvwKt8BiszKTwVo1u6+nTSp1Iq4jTgjMKdMBCqRdf8Fb4eTr9mqCmlIk4DzigGi3ZGeA6OW2Zy3GBGWSi1dPWRnqg9HKVUZGnAGcW+2nYS42IpzEiMyvtnJdlp6erD4QxtAc/jrT3kpUY2604ppTTgjKKizlVDLSYmOqtiZibbcRpo6w5dL6etx3VfKpJr+yilFGjAGVVDR1/U7t+ARz21ECYOHLCGCefnJofsnEop5YuwLjE90b1y91r6BpxRe/+TKkbnhuacFe5SPdrDUUpFmPZwxmC3Re+vKBwVoyvqO7DHxjArKylk51RKKV9owBnHwlEx+kBdB3NykrDppE+lVITpp844Fo6K0e5ECKWUirSwBhwRuUxEykWkQkS+5uX1O0Rkp4hsE5F3RKR4yOuzRKRDRO7x9ZyTSaI9loS4mJD1cHr6HRxt6mJBrgYcpVTkhS3giEgs8BCwDigGbhwaUICnjTHLjTGrgAeAB4e8/mPgFT/POalkJYWu2sDhxk6cBuZrD0cpFQXh7OGcBlQYYw4aY/qAZ4CrPXcwxrR5PE0GBmc4isg1wEFgtz/nnGwyk0NXMVoz1JRS0RTOgFMIVHo8r7K2nUREvigiB3D1cO6ytiUD9wLfCeSck0lWsj1k83Aq6joQgXk5GnCUUpEXzoDjbXr+sBotxpiHjDHzcQWY+6zN3wF+bIzpCOScACJyu4iUikhpfX29H80eX0K5Js6B+k4KMxJJtMeG5HxKKeWPcE78rAJmejyfARwbZf9ngJ9bj08HrhORB4AMwCkiPcBmX89pjHkYeBigpKQktMXIIiiUFaM1Q00pFU3hDDibgCIRmQtUAzcAN3nuICJFxpj91tMrgP0Axpi1Hvt8G+gwxvxMRGxjnXOyyUyy09YzQL/DGdSCaQ6n4WB9B2fPzw5h65RSyndhCzjGmAERuRNYD8QCjxljdovI/UCpMebPwJ0icjHQDzQDtwRyznBdw3iQZS2U1tLVT25qfMDnqW7upnfAqT0cpVTUhLWWmjHmZeDlIdu+5fH4bh/O8e2xzjmZeVYbCCbgVNS7FpPTgKOUihatNDDOnag2ENx9HE2JVkpFmwaccW6whxOCgJOTYidDl5ZWSkWJBpxxLlRr4lTUdTBfS9oopaJIA844l5HkShoIpodjjOFAfaeWtFFKRZUGnHEu3hZLSrwtqIrRDR19tHb3a9FOpVRUacCZADKT44KqGK0JA0qp8UADzgSQlWSnMYghtYp6DThKqejTgDMBBFsx+kBdB8n2WArSE0LYKqWU8o8GnAkg2DVxKuo6mJ+Xgoi32qdKKRUZGnAmgMxke9D3cDRhQCkVbRpwJoC81Hi6+hzc9MgHvLq7BofT9+LX7T391LT1aEq0UirqwlpLTYXGJ8+cjcMYnnz/CLc/uZmZWYnccuYcri+ZSXpi3KjHHqjvBDRhQCkVfRpwJoAku40vnL+A29fO49U9tTzx7mH+/a9lPLC+nHOLcrl8+TQuWpLvNfgcsFKitcqAUiraNOBMILbYGC5fXsDlywvYVd3Ki1uqeWXXcf5eVktcrHD2ghzWFuUyOyuJWdlJzMhMpKK+A1uMMDs7KdrNV0pNcRpwJqhlheksK0znviuWsL2qhb/tquHlXcd5s/zk5bRtMcKcnOSgFm9TSqlQ0IAzwcXECKtnZbJ6ViZfW7eYxs4+Kpu6qGzudv3Z1MWZusqnUmoc0IAziYgIOSnx5KTEs3pWZrSbo5RSJ9FxFqWUUhGhAUcppVREaMBRSikVERpwlFJKRYQGHKWUUhGhAUcppVREaMBRSikVERpwlFJKRYQY43up+4lKROqBI9bTdKDVy27etg/d5vl8pMc5QEOQTR6tTYHsF6prHvo8HNft6zWPte9UvObRXp/I/77H2ld/16NvH+vvYaTHCUCVMeYyH9s7NmPMlPoBHvZ1+9Btns9HeVwa7rb6u1+orjkS1+3rNY+171S85lBe93j6962/6+Cue6y/h0j8rt0/U3FI7SU/tg/d9pIPj0PJ1/OOtV+ornno83Bctz/nHG3fqXjNo70+kf99j7Wv/q5H3z7W30MkftfAFBlSiyQRKTXGlES7HZE2Fa9br3nqmIrXHY5rnoo9nHB7ONoNiJKpeN16zVPHVLzukF+z9nCUUkpFhPZwlFJKRYQGHKWUUhGhAWcUIvKYiNSJyK4Ajl0jIjtFpEJEfiIi4vHal0SkXER2i8gDoW11cMJxzSLybRGpFpFt1s/loW95cML1u7Zev0dEjIjkhK7FwQvT7/q7IrLD+j2/KiLTQ9/ywIXpmn8oInut6/6DiGSEvuXBCdN1X299hjlFxLfkglDnWU+mH+Bc4BRgVwDHbgTOBAR4BVhnbb8A+DsQbz3Pi/Z1RuCavw3cE+1ri/R1W6/NBNbjmnicE+3rjMDvOs1jn7uAX0T7OiNwzZcCNuvxD4AfRPs6I3TdS4BFwJtAiS/n0h7OKIwxbwNNnttEZL6I/E1ENovIBhFZPPQ4ESnA9R/vfeP6zfwGuMZ6+fPAfxpjeq33qAvvVfgnTNc87oXxun8MfBUYd9k54bhmY0ybx67JjLPrDtM1v2qMGbB2/QCYEd6r8F+YrrvMGFPuTzs04PjvYeBLxpg1wD3A/3rZpxCo8nheZW0DWAisFZEPReQtETk1rK0NjWCvGeBOa8jhMRHJDF9TQyqo6xaRq4BqY8z2cDc0hIL+XYvI90SkErgZ+FYY2xoqofj37XYrrl7ARBDK6/aJLdADpyIRSQHOAp7zGKaP97arl23ub3o2IBM4AzgVeFZE5lnfHsadEF3zz4HvWs+/C/wXrv+Y41aw1y0iScA3cA23TAgh+l1jjPkG8A0R+TpwJ/D/QtzUkAnVNVvn+gYwAPw2lG0Mh1Betz804PgnBmgxxqzy3CgiscBm6+mfcX3AenarZwDHrMdVwItWgNkoIk5chQHrw9nwIAR9zcaYWo/jHgH+Es4Gh0iw1z0fmAtst/5DzwC2iMhpxpiaMLc9UKH49+3paeCvjOOAQ4iuWURuAa4ELhqvXx6HCPXv2jfRvpk13n+AOXjcaAPeA663HguwcoTjNuHqxbhvtF1ubb8DuN96vBCoxJqAO15+wnDNBR77/BvwTLSvMRLXPWSfw4yzpIEw/a6LPPb5EvB8tK8xAtd8GbAHyI32tUXyuj1efxMfkwai/pcwnn+A3wHHgX5cPZPP4vrW+jdgu/WP7FsjHFsC7AIOAD9zBxXADjxlvbYFuDDa1xmBa34S2AnswPWtqSBS1xPN6x6yz7gLOGH6Xb9gbd+BqxBkYbSvMwLXXIHri+M262dcZeaF8bqvtc7VC9QC68dqh5a2UUopFRGapaaUUioiNOAopZSKCA04SimlIkIDjlJKqYjQgKOUUioiNOCoKUlEOiL8fo+KSHGIzuWwqjHvEpGXxqpOLCIZIvKFULy3UsHQtGg1JYlIhzEmJYTns5kTBRzDyrPtIvJrYJ8x5nuj7D8H+IsxZlkk2qfUSLSHo5RFRHJF5AUR2WT9nG1tP01E3hORrdafi6ztnxaR50TkJeBVETlfRN4Ukeet9VF+67F2yJvuNUNEpMMqcLldRD4QkXxr+3zr+SYRud/HXtj7nCgWmiIir4vIFnGtX3K1tc9/AvOtXtEPrX2/Yr3PDhH5Tgj/GpUakQYcpU74H+DHxphTgX8GHrW27wXONcasxlX9+Psex5wJ3GKMudB6vhr4V6AYmAec7eV9koEPjDErgbeBz3m8//9Y7z9mvSqr7tVFuKo3APQA1xpjTsG17tJ/WQHva8ABY8wqX7YrkAAAAdtJREFUY8xXRORSoAg4DVgFrBGRc8d6P6WCpcU7lTrhYqDYo3pumoikAunAr0WkCFel3DiPY14zxniuM7LRGFMFICLbcNWvemfI+/RxooDpZuAS6/GZnFhL52ngRyO0M9Hj3JuB16ztAnzfCh5OXD2ffC/HX2r9bLWep+AKQG+P8H5KhYQGHKVOiAHONMZ0e24UkZ8CbxhjrrXuh7zp8XLnkHP0ejx24P3/WL85cfN0pH1G022MWSUi6bgC1xeBn+BafyYXWGOM6ReRw0CCl+MF+A9jzC/9fF+lgqJDakqd8Cqu9VsAEBF36fZ0oNp6/Okwvv8HuIbyAG4Ya2djTCuuZZzvEZE4XO2ss4LNBcBsa9d2INXj0PXArdaaKIhIoYjkhegalBqRBhw1VSWJSJXHz5dxfXiXWDfS9+BaSgLgAeA/RORdIDaMbfpX4MsishEoAFrHOsAYsxVXtd8bcC38VSIipbh6O3utfRqBd6006h8aY17FNWT3vojsBJ7n5ICkVFhoWrRS44S1Smi3McaIyA3AjcaYq8c6TqmJQu/hKDV+rAF+ZmWWtTDOl+FWyl/aw1FKKRUReg9HKaVURGjAUUopFREacJRSSkWEBhyllFIRoQFHKaVURPx/zcxvTe+z8NAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot(skip_end=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.358833</td>\n",
       "      <td>0.371457</td>\n",
       "      <td>0.825000</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.374552</td>\n",
       "      <td>0.398303</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.364174</td>\n",
       "      <td>0.375110</td>\n",
       "      <td>0.835000</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.355115</td>\n",
       "      <td>0.358169</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.354921</td>\n",
       "      <td>0.355038</td>\n",
       "      <td>0.845000</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.349953</td>\n",
       "      <td>0.357888</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.371041</td>\n",
       "      <td>0.351703</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.347867</td>\n",
       "      <td>0.353313</td>\n",
       "      <td>0.845000</td>\n",
       "      <td>00:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.363351</td>\n",
       "      <td>0.361272</td>\n",
       "      <td>0.855000</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.355437</td>\n",
       "      <td>0.348741</td>\n",
       "      <td>0.850000</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.343979</td>\n",
       "      <td>0.354795</td>\n",
       "      <td>0.840000</td>\n",
       "      <td>00:53</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit(10, 1e-2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Collab Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.collab import *\n",
    "from fastai.tabular import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "user,item,title = 'userId','movieId','title'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://files.fast.ai/data/examples/movie_lens_sample\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "WindowsPath('C:/Users/bokhy/.fastai/data/movie_lens_sample')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = untar_data(URLs.ML_SAMPLE)\n",
    "path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>73</td>\n",
       "      <td>1097</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1255504951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>561</td>\n",
       "      <td>924</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1172695223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>157</td>\n",
       "      <td>260</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1291598691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>358</td>\n",
       "      <td>1210</td>\n",
       "      <td>5.0</td>\n",
       "      <td>957481884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>130</td>\n",
       "      <td>316</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1138999234</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating   timestamp\n",
       "0      73     1097     4.0  1255504951\n",
       "1     561      924     3.5  1172695223\n",
       "2     157      260     3.5  1291598691\n",
       "3     358     1210     5.0   957481884\n",
       "4     130      316     2.0  1138999234"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings = pd.read_csv(path/'ratings.csv')\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# It's very simple compared to Tabular Data\n",
    "# 1. Create databunch\n",
    "data = CollabDataBunch.from_df(ratings, seed=623)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Set the range of the value we want to get (in this case, its' the rating between 0 and 5)\n",
    "y_range = [0,5.5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Learner\n",
    "learn = collab_learner(data, n_factors=50, y_range=y_range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.643990</td>\n",
       "      <td>0.947816</td>\n",
       "      <td>00:44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.850718</td>\n",
       "      <td>0.641777</td>\n",
       "      <td>00:45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.650368</td>\n",
       "      <td>0.641316</td>\n",
       "      <td>00:46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(3, 5e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using Large MovieLens Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\bokhy\\.fastai\\data\n"
     ]
    }
   ],
   "source": [
    "print(Config.data_path())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "path=Config.data_path()/'ml-latest'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>307</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1256677221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>481</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1256677456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1091</td>\n",
       "      <td>1.5</td>\n",
       "      <td>1256677471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1257</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1256677460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1449</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1256677264</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating   timestamp\n",
       "0       1      307     3.5  1256677221\n",
       "1       1      481     3.5  1256677456\n",
       "2       1     1091     1.5  1256677471\n",
       "3       1     1257     4.5  1256677460\n",
       "4       1     1449     4.5  1256677264"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings = pd.read_csv(path/'ratings.csv')\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>date</th>\n",
       "      <th>N</th>\n",
       "      <th>url</th>\n",
       "      <th>g0</th>\n",
       "      <th>g1</th>\n",
       "      <th>g2</th>\n",
       "      <th>g3</th>\n",
       "      <th>g4</th>\n",
       "      <th>...</th>\n",
       "      <th>g9</th>\n",
       "      <th>g10</th>\n",
       "      <th>g11</th>\n",
       "      <th>g12</th>\n",
       "      <th>g13</th>\n",
       "      <th>g14</th>\n",
       "      <th>g15</th>\n",
       "      <th>g16</th>\n",
       "      <th>g17</th>\n",
       "      <th>g18</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?GoldenEye%20(...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Four Rooms (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Four%20Rooms%...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Get Shorty (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Get%20Shorty%...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Copycat (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Copycat%20(1995)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId              title         date   N  \\\n",
       "0        1   Toy Story (1995)  01-Jan-1995 NaN   \n",
       "1        2   GoldenEye (1995)  01-Jan-1995 NaN   \n",
       "2        3  Four Rooms (1995)  01-Jan-1995 NaN   \n",
       "3        4  Get Shorty (1995)  01-Jan-1995 NaN   \n",
       "4        5     Copycat (1995)  01-Jan-1995 NaN   \n",
       "\n",
       "                                                 url  g0  g1  g2  g3  g4  ...  \\\n",
       "0  http://us.imdb.com/M/title-exact?Toy%20Story%2...   0   0   0   1   1  ...   \n",
       "1  http://us.imdb.com/M/title-exact?GoldenEye%20(...   0   1   1   0   0  ...   \n",
       "2  http://us.imdb.com/M/title-exact?Four%20Rooms%...   0   0   0   0   0  ...   \n",
       "3  http://us.imdb.com/M/title-exact?Get%20Shorty%...   0   1   0   0   0  ...   \n",
       "4  http://us.imdb.com/M/title-exact?Copycat%20(1995)   0   0   0   0   0  ...   \n",
       "\n",
       "   g9  g10  g11  g12  g13  g14  g15  g16  g17  g18  \n",
       "0   0    0    0    0    0    0    0    0    0    0  \n",
       "1   0    0    0    0    0    0    0    1    0    0  \n",
       "2   0    0    0    0    0    0    0    1    0    0  \n",
       "3   0    0    0    0    0    0    0    0    0    0  \n",
       "4   0    0    0    0    0    0    0    1    0    0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load Moview Data\n",
    "movies = pd.read_csv(path/'u.item',  delimiter='|', encoding='latin-1', header=None,\n",
    "                    names=[item, 'title', 'date', 'N', 'url', *[f'g{i}' for i in range(19)]])\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   movieId                               title  \\\n",
       "0        1                    Toy Story (1995)   \n",
       "1        2                      Jumanji (1995)   \n",
       "2        3             Grumpier Old Men (1995)   \n",
       "3        4            Waiting to Exhale (1995)   \n",
       "4        5  Father of the Bride Part II (1995)   \n",
       "\n",
       "                                        genres  \n",
       "0  Adventure|Animation|Children|Comedy|Fantasy  \n",
       "1                   Adventure|Children|Fantasy  \n",
       "2                               Comedy|Romance  \n",
       "3                         Comedy|Drama|Romance  \n",
       "4                                       Comedy  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies = pd.read_csv(path/'movies.csv')\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27753444"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "58098"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(movies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>307</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1256677221</td>\n",
       "      <td>Three Colors: Blue (Trois couleurs: Bleu) (1993)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>307</td>\n",
       "      <td>4.0</td>\n",
       "      <td>832059248</td>\n",
       "      <td>Three Colors: Blue (Trois couleurs: Bleu) (1993)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>56</td>\n",
       "      <td>307</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1383625728</td>\n",
       "      <td>Three Colors: Blue (Trois couleurs: Bleu) (1993)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>71</td>\n",
       "      <td>307</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1257795414</td>\n",
       "      <td>Three Colors: Blue (Trois couleurs: Bleu) (1993)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>84</td>\n",
       "      <td>307</td>\n",
       "      <td>3.0</td>\n",
       "      <td>999055519</td>\n",
       "      <td>Three Colors: Blue (Trois couleurs: Bleu) (1993)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating   timestamp  \\\n",
       "0       1      307     3.5  1256677221   \n",
       "1       6      307     4.0   832059248   \n",
       "2      56      307     4.0  1383625728   \n",
       "3      71      307     5.0  1257795414   \n",
       "4      84      307     3.0   999055519   \n",
       "\n",
       "                                              title  \n",
       "0  Three Colors: Blue (Trois couleurs: Bleu) (1993)  \n",
       "1  Three Colors: Blue (Trois couleurs: Bleu) (1993)  \n",
       "2  Three Colors: Blue (Trois couleurs: Bleu) (1993)  \n",
       "3  Three Colors: Blue (Trois couleurs: Bleu) (1993)  \n",
       "4  Three Colors: Blue (Trois couleurs: Bleu) (1993)  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge two datasets by item and title\n",
    "rating_movie = ratings.merge(movies[[item, title]])\n",
    "rating_movie.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27753444"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(rating_movie)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Databunch\n",
    "data = CollabDataBunch.from_df(rating_movie, seed=623, valid_pct=0.1, item_name=title, bs = 24, num_workers = 2)\n",
    "# we need to tell to use titles as the item name instead of movieID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>userId</th>\n",
       "      <th>title</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>187135</td>\n",
       "      <td>How to Train Your Dragon (2010)</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4507</td>\n",
       "      <td>Me, Myself &amp; Irene (2000)</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>206005</td>\n",
       "      <td>Date Night (2010)</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51384</td>\n",
       "      <td>Hitchhiker's Guide to the Galaxy, The (2005)</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>77539</td>\n",
       "      <td>What's Eating Gilbert Grape (1993)</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.show_batch()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tricks for better results (what you should do in learner!)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#1. IN this case, use Y_range\n",
    "# Y range was the thing that made the final activation function a sigmoid\n",
    "# Sigmoid has an asymptote at 5, so, it's impossible to predict 5. But In real, many movies has 5 ratings\n",
    "# That's why we give a slightly higher end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#2. Add Weight Decay as wd\n",
    "# Generally, it should be 0.1 (the default is 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. n_factoris the width of the embedding matrix. (You should try 10,20,40,80 or bigger to find the best result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_range = [0,5.5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = collab_learner(data, n_factors=40, y_range=y_range, wd=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='99' class='' max='1040754', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.01% [99/1040754 00:31<93:03:43 1.7479]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEGCAYAAABYV4NmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO29eXicZ3nv/7lH+zqjffUmOU6c2I63LLZLFgiQBCgFSpu0HJZCKf1BS+khPaW0hVNOWwqlh1JOSdOSBBpIWwKUEiAJbUlCHGdx4t1OYlteJFv7NpJGI41mnt8f77zjkTSjmZHm1czI9+e6dFl6l5nn8UjznXsXYwyKoiiKkm5cmV6AoiiKsjJRgVEURVEcQQVGURRFcQQVGEVRFMURVGAURVEUR8jP9ALSSW1trVm7dm2ml6EoipIzvPTSSwPGmDonHntFCczatWvZv39/ppehKIqSM4jIOaceW11kiqIoiiOowCiKoiiOoAKjKIqiOIIKjKIoiuIIKjCKoiiKI6jAKIqiKI6gAqMoiqI4ggqMsmie6xjkSNdoppehKEqWogKjLApjDL/78AHueeRQppeiKEqWogKjLIrT/RP0jU3xSs8Yp/rGMr0cRVGyEBUYZVHs6xiMfP/o4e4MrkRRlGxFBUZZFM+dHqTJXcz166r5kQqMoigxUIFRUiYUMjzXMciuthretqWJk33jvNarbjJFUWajAqOkzGt9YwxOTLOrvYbbNzXhEnj00MVML0tRlCxDBUZJmX2nrfjLrvYa6iqKuLGthkePdGOMyfDKFEXJJlRglJTZd3qQVdUltFaVAvCWLU109E9wolvdZIqiXEIFRkmJUMjw/JkhdrXVRI7dfk0jeS7hR0fUTaYoyiVUYJSUON7tZXQywO722sixmvIidrfX8OhhdZMpinIJFRglJaLjL9G8dUsT5wZ9HLvozcSyFEXJQlRglJTY1zFIW20ZDZXFs46/6epG8l3CDw+rm0xRFAsVGCVpZoIhXjgzxI1zrBeAqrJCdrXX8NSr/RlYmaIo2YgKjJI0Ry96GZ+aYXcMgQG4urmSjoEJgiGNwyiKogKjpIAdf7mxLbbAtNeVMz0TomvYt5zLUhQlS1GBUZLm2dMDbGgop7a8KOb59rpyAE71jS/nshRFyVJUYJSkOXJhlB1rquOeXx8WmNP9KjCKoqjArBhOdHt57KhzXY1DIcPoZIC68sK417hLC6gtL+J034Rj61AUJXdwTGBE5H4R6RORo3HOu0XkhyJySESOicgHwse3isi+8LHDIvKrTq1xJfHV/z7FHzxy2LHHn5iewRioLClY8Lr2ujJOqQWjKArOWjAPArcvcP6jwHFjzLXALcCXRKQQ8AHvNcZcE77/yyLicXCdK4KTfWN4/TNMTM048vhev/W4FcX5C163vr6cU33jWtGvKIpzAmOMeRoYWugSoEJEBCgPXztjjHnNGHMy/BgXgT6gzql1rgQCwRBnBiy3VI/X78hzeCcDAFQWJ7JgyhmdDDA4Me3IOhRFyR0yGYP5KrARuAgcAT5ujAlFXyAi1wOFwOl4DyIiHxaR/SKyv7//8izyOzfoIxC0LIaeUWcEZixswSRyka2vDwf6NZNMUS57MikwbwYOAs3AVuCrIlJpnxSRJuCfgQ/MFZ5ojDH3GWN2GmN21tVdnoZOdFpwt0MCY1swiVxk7bbA9GugX1EudzIpMB8AvmcsTgFngKsAwkLzI+CPjTHPZXCNOcGpvktzWHpGJx15Dq8/ORdZU2UxJQV5WgujKEpGBeY88AYAEWkArgQ6woH+7wPfNMZ8J4PryxlO9o3T4imhqrTA+RhMAheZyyW015dpLYyiKCzs71gCIvIwVnZYrYh0AZ8BCgCMMfcCnwMeFJEjgAD/yxgzICLvAW4CakTk/eGHe78x5qBTa811TvaOc0VDOb3eKcdjMIlcZGAF+vefHXZkHYqi5A6OCYwx5u4E5y8Cb4px/CHgIafWtdIIhgyn+8fZs74Gl4hzMRh/gJKCPAryEhu97XXl/ODgRSang5QU5jmyHkVRsh+t5M9xuoZ9TM2EuKK+gkZ3sWMWjHdyhsqS5D6P2JlkHQPqJlOUyxkVmBznZK/1Jr6+oZzGymIGJ6aZmgmm/XnGpgJUJAjw22jTS0VRQAUm5zkZfhNfX19Oo9uaMtnnnUr783gnZ6hMIv4CsLa2FJdoqrKiXO6owOQ4J/vGaKgsorK4gKawwDgRh/H6AwkzyGyK8vNYXV2qxZaKcpmjApPjnO4b54r6CoAogUl/LcyYfyZhDUw07XXlmqqsKJc5KjA5jDGGk33jkaB6Q6UlME4E+r2TgaRSlG3W15fr+GRFucxRgclhLo768U0HuaLBEpiK4gLKi/LTXmxpjEnJRQY6PllRFBWYnOZkr9UixnaRAY6kKk/NhAgETWousvoyQKdbKsrljApMDmOnAV8RdpGBFYdJd5A/2UaX0WiqsqIoKjA5zMnecWrLC6kquzTGuKEy/RZMpNFlCi4yT2khteWFOj5ZUS5jVGBymJN9Y5EAv02Tu5j+8SlmgnEnHKTM6GR4FkwKFgxAW125jk9WlMsYFZgcxc4gi46/gBWDCYYMA+Ppmyg55rddZMlbMABXNlTwSreXyen0dxZQFCX7UYHJUfrGphjzz0QyyGycqIXxhjspu5PsRWbz1i1NTEwH+dGR7rStRVGU3EEFJkexg+fr62YLjBO1MJFZMClaMNevq6atroyHXziftrUoipI7qMDkKHaK8vp5FkwJkN52MfYsmFSC/AAiwq9dv5qXzg3zas9Y4hsURVlRqMDkKCf7xnGXFFBXXjTreFVpAYX5LnrTWGzp9QcoyBOK8lP/dXnn9lYK81xqxSjKZYgKTI5yfsjHutoyRGTWcRFJey2MdzJAZXHBvOdKhuqyQm7f1Mj3Xu7CH9Bgv6JcTqjA5ChDE9PURNW/RNOY5lqYMf9Myu6xaO6+fjVe/ww/1mB/WhiemOY9//Q8F0fS39RUUdKJCkyOMuIL4C6N/abf6C6m25vOLLLUGl3O5ca2atbVarA/XRzqGuGZUwMcOD+S6aUoyoKowOQoI75pqkrjWDDuYnpHpzAmPZ2MbRfZYhER7r5+FS+eHY4kJyiLx46vjYaz+xQlW1GByUGmZ0JMTAepimPBNFUWMx0MMTSRnmJLy0W2eAsG4F3bWynIEx5+oTMta7qc6Q1PLFWBUbIdFZgcZMRnCYc7rgWT3lRlrz9ARdHiLRiAmvIi3nxNI999uYupGQ32LwV7HIPdI05RshUVmBxk2Ge9scSzYBrd6S229E4u3YIBy4oZnQzw7KnBNKzq8qV3VF1kSm6gApOD2BZMvBhMpF1MGmphAsEQk4HgkmIwNrvX11BRnK+tY5ZIj8ZglBxBBSYHsS0Yd5zU4dryIvJcEvmkuxTsKv6lZJHZFOXn8caNDTxxrIfpmfR1e77csGMwXhUYJctRgclBIhZMnDqYPJfQUFGUlhhMpA/ZEupgorlzcxNe/wz7OtRNthgCwRCDExrkV3IDRwVGRO4XkT4RORrnvFtEfigih0TkmIh8IOrc+0TkZPjrfU6uM9cYmVw4BgPQ4C6mJw21MJFhY2lwkQH8whW1lBfl8xN1ky2KvrEp7OxzFRgl23HagnkQuH2B8x8FjhtjrgVuAb4kIoUiUg18BrgBuB74jIhUObzWnGHYN01hvouSgry416SrXcxiG13Go7ggjzdsrOfxYz1pHYp2uWAnbrRWlajAKFmPowJjjHkaGFroEqBCrCZX5eFrZ4A3Az81xgwZY4aBn7KwUF1WjEwE8JQs3BussbKEnlH/kostbRdZOmIwNndsamLYF+D5Mwv9aiix6AsH+Dc0VOCdDKStmFZRnCDTMZivAhuBi8AR4OPGmBDQAkRX5HWFj81DRD4sIvtFZH9/f7/T680Khheo4rdpqSrBNx2MJAQsloiLLE0WDMAtV9ZRWpinvckWgZ1BdkVDOSED41MzGV6RosQn0wLzZuAg0AxsBb4qIpVArI/mMT+qGWPuM8bsNMbsrKurc26lWcTIZADPAvEXgLU1pQCcGZhY0nNFXGRptGCKC/K49SrLTRYM6SfwVOjx+inIE9bVlAEah1Gym/S9ayyODwCfN5adf0pEzgBXYVkst0Rd1wo8ueyry1JGfNOsqy1b8Jq14fNnBybYsWbx4SvvZAARKCtM76/KnZua+NHhbl44M8Su9pq0PnY6CQRDfHPfObyTAYIhQ9AYXALv272W+oriZV9Pn3eK+oriyAeM0ckArRqdVLKUTAvMeeANwM9FpAG4EugATgF/ERXYfxPwqcwsMfsY9gXYnsBFtqqqFJfAucGlWTBe/wwVRfm4XKnPglmIW66so7jAxU+Odme1wOw9NcDnHj0OgEusFPBA0DA0EeAv37l52dfTM+qn0V0ccVmqBaNkM44KjIg8jGWJ1IpIF1ZmWAGAMeZe4HPAgyJyBMst9r+MMQPhez8HvBh+qD8zxmhEGDDGMOoL4EkgMIX5LlqqSjgz6FvS83n9gbTGX2zKivK5ZUM9Pznaw2ffdk3aBSxdHO/2AnDoT98UGY/wqe8d4bsvdfGJ266gvnJ5rZher5+NTZWRIlsttlSyGaezyO42xjQZYwqMMa3GmK8bY+4NiwvGmIvGmDcZYzYbYzYZYx6Kuvd+Y8z68NcDTq4zl/BNB5kOhhLGYADW1pRxdokxGO/kDBVpqoGZy5s3NdA/NsWxi15HHj8WR7pG+dbz55K+/pXuMVo8JbNm7/zWTW3MhELcv/esAyuMjzGGHq+f+sqiKIHRIL+SvWQ6yK+kyHCkD1niN/11tZbALCWV1esPpDXAH8316yzX2Mvnhx15/Fg8+OxZ/vQHx5Lu6Hyi28vGpopZx9bWlnHHpia+9dy5Ze1oPD41g286SGNlcURg1EWmZDMqMDnGSDjtOJGLDCwLZmxqhsElzIVZ6rjkhWh2F9NQWcRL55ZPYM4PTRAMGTr6E1t2/kCQjoEJNjZVzjv3kZvbGZua4dvPL9+UTnvQWKO7mPKifFyiAqNkNyowOcZIpFV/YoGxM82WEuj3Ti5tXPJCiAg71lQtqwVzfsiKSb2WxGTNU33jBEOGqxrnC8zmVjd71tfw9WfO4A8sz3ybnlGrB1lDZTEiQmVJgQqMktWowOQYtossmRjMmkgtzOID/ZaLzBkLBmD76iq6hicjFepO4g8EI52IkxEYO8A/10Vm89s3r6d/bIrvH7iQvkUugG3BNIQTC9wqMEqWowKTY4ykIDCrqkvJc8miA/2hkGF8yjkXGcD2cI3OclgxnUOXhPa13vGE17/SPUZJQR5ramLXHO1ZX8Omlkrue7pjWQpG7Sr+RhUYJUdQgckxIjGYksQusoI8F61VJZxZpItsfHoGY9JbxT+Xa5orKcxz8fL5Eceew8Z2jzW7i5OyYE50e9nQWEFenBRqEeEjN7dzZmCCnx7vSetaY9Hr9VNZnE9JodXkVAVGyXZUYHKMYV+AssI8CvOTe+mWkqocmQXjoIusKD+Pza3uZQn0nwvXBN12dQPnh3xMTsePnRhjONHj5eo47jGbOzY1UVteyGNHnRcYu8jSprKkYFmz2BQlVVRgcowR33RSGWQ262rLODfoW1Sqsl1jUVnibMOH7as9HLkwmnTq8GI5P+SjvCifG9tqMMYK4sej1zvFiC8QM8AfTZ5LuLGthmdPDzre2bh3bCoSfwHLgtFCSyWbUYHJMYZ901SVJW9RrKkpZXxqhoHx1FOVx9I8bCweO9ZUMT0TcrzgsnPIx6rqUjY0WFbJQm6yE5EA/8ICA7C7vZa+sSlOJ5H6vBR6R/3zBGZUW/YrWYwKTI4xMhlIKv5iE2l6uYg4jDfcSdmpSn6b7avDgX6H3WTnhnysri5hbU0phXmuhQWmxxKYqxK4yAB2h3upOTkGOhgy9I9PRQL8YAl/IGiYXKY0aUVJFRWYHGPEl7hVfzR2W/fFtO2PxGAcdpHVVxbT4ilxNJMsFDJ0DvlYU1NGfp6LtrqyBBaM1SImGettTU0pze5i9p0eSOeSZzEwPkUwZGioLIoc02p+JdtRgckxkhk2Fk1rVQn5i0xVXi4XGVhuspfPOZdJ1j8+xdRMiFXVVm3QhoaKBVOVrRYxid1jYGWT7WqvZd/pQUIOpSvPrYEBFRgl+1GBySFCIcPoZCCpPmQ2+XkuVlWXLslFVu5gmrLN9tUeerx+Lo5MLulx/uj7R/irx16Zd9zOIFsdFpgrGyu4MDIZEdFo/IEgHf3jcQssY7GrvYZhX4BXehKnPy+GntFLbWJsIgKzxKmliuIUKjA5hNcfwBhwp2DBgOXCObuIan7vZIDSwjwK8pz/NdmxphpgyenKT7/Wz/dfvjAv8G3XwKyJsmAATsbIJDvZO07IJBfgt9m1hDjMS+eG+Lv/OrngNb1ziizhksDYHwQUJdtQgckhhiN9yFJzWa2tKePsYOpdlcf8M8viHgMrmF5c4FpSHMYYQ593yrKERme3njk/5MMl0OwpAWBDQzkAJ2PEYVLJILNp8VjJA4uJw3z7+U6+9NPXFnRj9nqnyHMJNeUag1FyBxWYHOJSq/7ULJh1tWX4poP0j02ldJ81bGx5hp4W5Lm4ttWzpEyyEV+A6WAIgP1nZ8+nOz84QbOnJFKguqqqlOICF6/2zLdgTvR4KSnIi7jTkmVXey3PdwwxE15Dstjuyx8d6Y57TY/XT1150ayuAitdYM4OTPCd/Z0cv+hN+f9UyQ5UYHII29fuTtWCqV1cJpnXH3A8RTma7WuqOHbRu+juxL1jl6yWua6280O+WYLhcglX1Fdwsi+2BXPlAi1i4rGrvYaxqRmOpljPY3e7/vECAtPr9dPgnj09046NrVSB+Ysfn+CeRw5z51d+zqbPPs47/34vf//kqUwvS0kBFZgcYtEWTM3iamEsF9nyWDAAO1ZXMRMyHO4aXdT9dqfkiqL8hAIDVhzm1TlBeWMMr/SMpRTgt9nVFo7DnE4+DjPmDzAwPk2Tu5hjF71xRyv0jPppjEpRBquLQEVx/oqs5jfG8PL5EW7b2MDf3rWVX79hDRNTQb7w2KuReJSS/ajA5BCLjcE0e4opyBPODqYW6PdOBhztpDyXa1d5ADjctbh0Zbvl/21XN3Ci28v4lBX8ngh3MlhdM1dgyukbm4p0qAbLFTXiC6QUf7GpqyhiQ0M5z6YQh7GTLz70ujYgvpus1zu7it9mpTa87BqeZGB8ips31PL2rS38yVuv5ku/ci2QmoArmUUFJocY9U0jknplfX6ei1VVpSnXwnj9M44NG4tFXUURze5iDi3SgukLx5ju2NRIyMChTkuoOodnpyjbbGi0W8ZcisP8/DVLHBL1IIvH7vZaXjw7xPRMcjED26rcs76Gras8Md1kk9NBvP6ZuAKzEi2YA+HXblu4ywPA1U2VuEsKUhJwJbOowOQQw74A7pKClGMDYMVhUonBGGMsC2YZYzAAW1o9i7Zger1+3CUF3NhegwjsP2u5yebWwNjM7Un2fMcgf/yDo1y7ysPWsDWVKje21eAPhDjYmdwebNFfU13GWzY3cfSCl/NzLM25c2CiWakWzMHzIxQXuLiy8ZKr0uUSdoUbiyq5gQpMDpFqFX80a2tS66o8PjXDTMhEMpWWiy2r3Jwb9M1yWyWL5UYqorK4gCsbKth/zsok64zUwMweHNYcnm3/Wu8Yxy96+dA39rOqqoQH3n9d0uMQ5nJjWzUiJP0p+8zgBI2VxZQU5nHH5kZgvpssVpGlzUoVmAOdw2xucc+rwdq9voau4cl5IqxkJyowOUSqfciiWVdbymTUyOBEdA5ZFfWtVaml6i6Vra2W5bAYN1mvd4r6CutNeMeaKg6eHyEYMpwf8lFZnD8v+05EuKKhnH2nB3nv/S9QUZzPP3/wBqrLFifiAJ7SQq5t9fCfJ3qTuv7coI+1tdb/cWtVKdeu8vCjIxcj58f8Ab7w+Cvku4T2uvJ5969EgZmaCXLsoneWe8zGbiyqbrLcQAUmhxiZnMazSIvCFooLI8l98js/FHbd1CyvwGxqdQNwOEkXUzR9Xj/14UyrHWuqGJua4bXeMc4N+uYF+G2ubKjgZN84wVCIb37whkgh5lJ46xbL1ZVMzOvswARro0Yyv2VzY8RNNuYP8N77X+BI1yhf/bXtMS2YyhUoMCe6x5ieCbEthpuyva6cuooidZPlCCowOcTwRGDRLrKWKuuN88JIcimekbjFMgtMZXEBbXVlKVswoZChL2og186o1jOdQ7557jGb69ZWU1GczwMfuJ719fMthMVwx+YmYOHCSbDqjAYnpiN1SmBNyAT41/3nZ4nL7ZsaYz6Gu6SAqZlQyrVDXcM+Liyx75tTHAh3c9i6er7AiAi725dnwJuydFRgMsDUTJDP/OAox1MsyEt1mmU0TeFPv8k2kzw35KOqtGDZg/wA1y4i0D/sm2YmZGiosCyYVdUl1JYX8eLZIbqGJyNdlOfyrh2tvPwnb1x0UD8WLZ4Stq/28KPDCwvMuXCKcrQFs6q6lGtb3fy/n51OKC5AJI081Uyy3/+3Q7z7a88yMZV9fcwOdo7QWFlMkzu2Nbm7vYaB8akFJ5Iq2YFjAiMi94tIn4gcjXP+HhE5GP46KiJBEakOn/uEiBwLH39YROb7BnKYE91jfGPfOd5977P89yvJ+eqnZ0JMTAcXHYOpKC6gsjifC8PJCUxnjMLE5WJLq5u+salIcDsZ7NiSbcGICDvXVPFfJ/qYDoYWdPU50czzLVuaOd7tpaM//pvgmXCKsh2DsfnlnasoyJOE4gLRDS9TE5iO/gkujvr58n++ltJ9y8GB8yMLCv7u9loAdZPlAE5aMA8Ct8c7aYz5ojFmqzFmK/Ap4CljzJCItAC/C+w0xmwC8oC7HFznsjMQrtcoL87nQ9/Yz4N7zyS8Z2TSruJfvEXRUlWavAUz6GN1TWy3ktNsiQT6k7di7DYx9VGpvDvWVEWKLZdbLO8MZ4Qt1P4lOkU5mvfcsJqX/+SNCcUFFtePbHI6yMD4FOVF+dy/92zKljTAsYujvP6vn0y5v10iBsenOD/kY1sM95jNqupSWqtKNNCfAzgmMMaYp4GhhBda3A08HPVzPlAiIvlAKXAx5l05ysC49Uf50Adv4A0bG/jsD4/zmR8cXbCh30i4in+xLjKAFk9xUn73QDDEhZHJSGv75eaa5kryXRIplEyGvshArkvtVHasvZSFtNwC0+QuYeeaKh5dwE12dmCCJreVohyNiCRdTLsYgekKF57e8+Yr8ZQU8EffP0IwxUFpPzh4kY6BCY5dXFxRbDwOxiiwjMXu9hqe6xhKed3K8pKUwIhIu4gUhb+/RUR+V0TS4rQWkVIsS+e7AMaYC8BfA+eBbmDUGPPEAvd/WET2i8j+/v7+dCzJcWyBWVVdyr3v2cGHb2rjG/vO8S8vdsa9Z3jCsmAW6yIDq1V9MgJzcWSSYMgse4Dfprggjw0NFSn1JLNdZHUVlwRmU7ObwnwX+S6JxKCWk7dsaeKVnrG4sYKzgxNLztJbjMDYnQ02tVTyx2/dyMHOEb79wvmUnvfJV/sA0p4ocOD8CHkuYXOLe8HrdrfXMjoZiIxWULKTZC2Y7wJBEVkPfB1YB3w7TWt4G7DXGDMEICJVwNvDz9EMlInIe+LdbIy5zxiz0xizs66uLk1LcpaB8WkqivIpLsgjzyX80Z0bqa8oWnAWysik3YdsKRZMCWP+mYT+ejuDLFMWDMC1q9wc7hpJOlOo1+unqrSAovxL1kBhvotrW93W2OhlGJo2lzs2NSES3012dtDHutqluSHtZqSpTLXsCsfhVlWV8ktbW9izvoYvPPYKfWPJxbwujExG2ussdQLpXA52jnBVY8U8q24uu7QeJidI9q8uZIyZAd4BfNkY8wmgKU1ruIvZ7rHbgDPGmH5jTAD4HrA7Tc+VFfSPT1FbMbsz7samSk50xx+3a1e2L9WCAehOkKpsT3/MlAUDViaZ1z+TdIPOXu9UzF5dn/3Fa/j8u7ake3lJ0egu5ro11TGzyUYnAwxNTM/KIFsMlRELJvlssM4hH0X5LuoqihARPvf2TUwFQtzzncMRS3khbOulKN+VdNJIMgRDhoOdCwf4bRoqi1lfX66B/iwnWYEJiMjdwPuAR8PHlpy/KiJu4GbgB1GHzwM3ikipiAjwBuDEUp8rm+gfm6K2fLYlcnVzJaf6xuI2SbzUSXnxFowtMImKLc8P+SjMd9FQkbnkPTvQn2y6cv+Yf1aA3+aaZjc3htvoZ4K3bGni1d6xeZMz7bb8a5YoMAV5LsoK81LKIuscmqS1qgTrzwva6sr59Fs28sypAV7/pSf51xfPE1ogtvHkq/20eEq4dpUnrS6y0/3jjE/NJIy/2Oxur+GFM0MEdBhZ1pKswHwA2AX8uTHmjIisAx5a6AYReRjYB1wpIl0i8kER+YiIfCTqsncATxhjIiXPxpjngUeAl4Ej4TXel/SOcoCB8Slqy+dbMIGgieuvH/EFKMgTShO4DhaiNcliy3ODE6yuLsW1iKaa6WJDQznFBS4OdSYXh+n1TkVqYLKJOzY1IjK/6NJuPLpUFxmk3i6mc9g3ry7ofbvX8ujv/ALr68v5X989wrvufTZmAH9qJsjeUwPcelUdrZ4SLiZZuJsMB8/bAf7kwrs3rKvBNx3UOEwWk5TAGGOOG2N+1xjzcDhGUmGM+XyCe+42xjQZYwqMMa3GmK8bY+41xtwbdc2Dxph5KcjGmM8YY64yxmwyxvwPY0x6cyEzzMDYfIG5Ojx/5HicPxa7yNL+1LkY6sqLKMiThH7zc4O+jMZfwBoxcE2zO6lU5WDI0D8e20WWaeori9ndXsO/vtg5yzq158CkoxVPqu1iOod8rIrRY25jUyX/9lu7+JtfuZbOIR/vvnffvOFe+88O45sOcsuGelqqSujx+tM2zvhA5zCVxfmRAXmJsCv9D5xfXPdtxXmSzSJ7UkQqw4WQh4AHRORvnF3aymRqxprtMVdg1tWWUVzgivtpzOqkvDSvpMslNLqLF/SbG2OsN6AMCwxYBZfHLo4mfAMbnJgiGDKzUpSzid+6qZ3uUT/ffT5kXxkAACAASURBVLkrcuzcoJWiXFyweIvUJhULZnQygNc/w6rq2FXyIsI7t7fy3d/eTSAY4m//6+Ss80++2kdhnovd62to9pQQDBl601ALc6pvjH8/cJFfuKI2acu52V1MXUVR0qMRlOUnWReZ2xjjBd4JPGCM2YEVjFdSZHDcCqLWVcwff3tlY2XcordhXwBPyeLjLzYtnpIFLZjBiWkmpoPL3uQyFte2evAHQrMGgsWiL5yiHCsGkw287oparm118/dPnoqI5ZnBiSUH+G1SGTpmjy6IZcFEs6amjF+/YQ3/+mInp6O6Efzs1X5uaKumtDCfFjumt8RAvz8Q5GPfPkBJYR6feds1Sd8nImxd5VGByWKSFZh8EWkCfoVLQX5lEdg1MHOD/ABXN1VwoscbMzW3z+tfUht5m+YEAhNJUc4Cgdlid1ZO4CbrjRRZZqfAiAgfe/0VdA5N8h+HrJrhswMTs5pcLoVUXGR2kWUyFurHXr+e4nwXf/34q5F7T/WNc/MGqxwg2aSRRPzFj0/wSs8YX3r3tSm/hltXeTgzMLGo+UGK8yQrMH8GPA6cNsa8KCJtwMkE9ygxiAhMjID0xqZKRnyByARDm65hH2cHfexcm1x2zUK0eCy/ebzMG7tN/+o43YeXk7U1ZVQU5XPkwsKBfntUcra6yABu21jPxqZKvvqzUwxPTDPsC7A2TSKeiovs0pyfxGMJasuL+PBN7fzkaA8vnx/myVetQuZbr6oHiFgwSwn0P3a0h2/uO8dvvm5d5HFTwW7pr1ZMdpJskP87xpgtxpjfDv/cYYx5l7NLW5kMjIVdZOXz3wwjgf45brKfhf+wb7ky9T/AubR4SggZ5gVvbc4N+hBJ7g3IaVwuYVOLO6HA9Hr9iDAvrpVNiAgfu3U9Hf0TfO2p0wBps2DcJQX4poNJpet2DvuoKMpPelLph163jtryIj7/41d48tU+VlWX0BZed0lhHjVlhZHCzVTpGvbxB48cYkurm3vefNWiHmPLKg8iKjDZSn4yF4lIK/B3wB7AAM8AHzfGdC14ozKP/oiLbP6b4VVhgTnR7eUNGxsix598xfrDbq9b+htSc9SnzljTKs8P+WisTE/wOR1saXXzwN6zTM+E4o4x7vVOUVNW6EhX5HRyx6ZG1teX8/VnrOam6UhRhqiOypMBahKIbOeQj9bq0qSzEcuK8vn4bVfwJ/9+FJfAr9+wZta9iVyu0Xz/QBcvnRumzztF//gU5wZ9hAz83d3bFj2iurwonw31FZpJlqUk+6o+APwHVuuWFuCH4WNKigyMT1FWmBezFUZ5UT5rakpnpSr7A0H2nh7g9VfWLylF2SaR3/z8YOba9MdiU4ub6WCI13rjdzno8/ojo5KzGZdL+Oit7ZEGjen6f06lH1nn8CSrUrRO77puFWtrSgkZuPWq2e2YWpLsbzcTDPHJ7xzm3w9c5Nygj7LCfG66opZ/fO/OJRebbl3l4VAKbYWU5SNZgakzxjxgjJkJfz0I5EbjryxjYHw6ZvzFZmPj7JYxz58Zwh8Iccsi/NOxSOQ3Pzfky4oAv82lQH98N1nvmD+r4y/RvG1LM2tqSmnxlKTNSkxWYIwxdMUoskxEQZ6L//32TexcU8WuttpZ52wLJtGbe9+YlUr+6bds5PFP3MRDH7qBL9+1LdJTbClsXe1hxBdIuq2QsnwkKzADIvIeEckLf70H0CZAiyBWkWU0VzdXcnZwIjJp8Gev9FGU72JXmtqdlBTmUV1WGPNTp296hv6xqSV/okwnq6tLqSxeONAfrw9ZNpKf5+L//dp2vvDL6euPVjlHYCamZnjp3NC8aZUD49P4A6GULRiAmzfU8chv755nebdUleCbDkbGScSjOzw8rtGBrtZbI4H++M1ilcyQrMD8BlaKcg9WC/1fxmofo6SI1SYmfrrxxqZKjIFXesYwxvCzV/vY3V6T1phIsyd2saXd5DIbiixtRITNrW6OXIjtY58JhhgYn8raGphYbGpxs2d9beILk8RdYoVS/+GpDt7ylZ+z+bOP866v7eNvfjp7WmVnCinKydLisf7fE7nJ7OmkToxN2NBQQWlhXqTVjJI9JJtFdt4Y84vGmDpjTL0x5pewii6VFInVhyyajU0VgBXoPzMwwblBH69Pk3vMJl6x5fksaNMfi80tHl7tGWNqJjjv3ODENMZkd4qy09RXFlNSkMfhrhE8pQV87Nb17FhTxWNHe2a5rjod+ADR4rEeK5HAdI9a55vc6c9OtOfHaCZZ9pFUFlkcfh/4croWcjkQCIYY9gUWFJgWTwmVxfkc7/YyFe5dlY705GiaPSU8c3IAY8ysxAHbgsmmGAzA5hY3gaDh1Z6xSJdlm0iRZQ4E+Z2isriA/X98W2S+EEDr/k7+4JHDHL3gZXM4jmWnE6czBb2lKrlq/u5RP6WFeZH5Nelm2+oqvv5MB/5AMGsyIJWljUzOXKvdHGUoPGtjoSC/iIRnw3h58tU+1teXp91l1eIpYWI6OC8ofG7QR2Vx/pLGMjuBHeiPFYfpjbSJuXwtGLDSifOienjdtrGBPJfw+LGeyLHOIR81ZYWUFqbvTb6qtIDiAlfCVOWeUT+N7uK0ZELGYusqD4Gg4VicVktKZliKwGhOYIr0hyvO6xaIwYAV6D/R7eX5jiFuvTL9yXqRHlJz3hTODfkyOmQsHq1VJXhKCzgSI5Ms29vEZIrqskKuX1vNY9ECM2zVwKQTEUkqVfni6CTNDrjHbOwW/+omyy4WFBgRGRMRb4yvMayaGCUFBhYosoxmY1Ml/kCI6WBoUe0zEtEcJ1X5/OAEa7KgRcxcRCwfe6xU5T6vH5dATRr6tK003nxNA6f6xiPNKjuHUq+BSYbmJATGtmCcoqGymCZ3sQpMlrGgwBhjKowxlTG+KowxzjhTVzAD4U7KiQTGbhlTXpTPzjXVaV9HpNhy+FLdQDBk6BqezEoLBqw4zGu9Y/gDswP9vV4raSI/y6v4M8GbrmkE4PFjPQRDhosjk45kCLZWLVzNPxMM0Tc25UgGWTRWZ2VNVc4m9K9yGVmo0WU0VzSUk+8SfmF97aJbaCxEbXkhhfkuLo5esmCOXBhlJmSyLoPMZkurm5mQ4ZWe2RX9VpGlusdi0ewpYUurm8eP9dI9OslMyCRs07+o53GXhGts5mf5gdUeKRgyjlowYAlM59BkZFqoknlUYJaRgbEpigusGeoLUZSfx1fu3sY9t1/pyDrm+s190zN88juHqKsoinzqzTY2tcQO9Pd5py7rFOVEvPmaRg51jrD/rPXJPt6gsaVgZ5LFs2LsIksnYzAAb9jYQElBHm/9ys+5/5kzkZY8SuZQgVlG7BqYZDJp7tzcRHtduWNriS62/NyjxzndP86Xf3VrWmbOOEGLp4TqskKORM2GefHsEKf6xrOqMDTbeHP4A8P9e60Gm05YMPGSRmx6HKzij2Z9fTlPfOImrltXzZ89epx3/v1ejl1cuBO34iwqMMvIwPj0vEmWmcIutvzR4W4efqGTj9zcntbq8nQjYrXutwP9Ry+M8hsPvEhrdQkfvXV9hleXvayvL6e9rozDXaOIXIq/pZPmBJMtbcvG6RgMWEWkD7z/Ov7u7m1cGJnkF7+6N+HAOsU5VGCWkURV/MtJs6eEvrEp/vB7h9m6ysPvv3FDppeUkC0tbk72jXP0wijvvf8FKksK+NaHbsia/9NsxbZimiqLHYnpNbqLcUl8F1nPqJ+SgrykZ9AsFRHhbdc285OP30QwZHgqPE9JWX5UYJaRbBMYAMLzOLJ9lgrA5lY3wZDhV/9hHy4RHvrQDY60Hllp3L7JEph018DYFOS5aKgspiteDMbrp8nBIst41FUUsa62jKPqJssY2f+uskIIhgxDE9MJiyyXi42NlYjAn79zc87EMDaHA/15LuGfP3h92gZ2rXQ2t7hpqy2LpL87Qbz+duB8DcxCXNNcydELWt2fKbSWZZkYmpgmZBKnKC8Xm1vdHPrMm6gsXh63RTpochdzz5uv5KYr6tjo4JvlSkNE+MHH9lCU71yPrmZPCQfi1KB0j0yyqz0z8b1NLW4ePdzN8MQ0VVmawLKSUQtmmbDbxGSLiwzIKXEB643yo7eujzRvVJKnorjAkfiLTUtVCT2j/nmpwcGQoXcZiizjYVu92qMsM6jALBPJtolRlFykxVNCIGgiH6RsBpapyDIe1zRblq7GYTKDYwIjIveLSJ+IHI1z/h4RORj+OioiQRGpDp/ziMgjIvKKiJwQkV1OrXO5uCQwaqYrK49LtTCzxxYvZ4pyLDylhbRWlXB0gYmoinM4acE8CNwe76Qx5ovGmK3GmK3Ap4CnjDFD4dN/CzxmjLkKuBY44eA6l4Vk28QoSi5iJ1zMbeVzaZJl5rL9NjW71UWWIRwTGGPM08BQwgst7gYeBhCRSuAm4Ovhx5k2xuR8pdTA+DSF+S4qijSvQll5rKkppb6iiOc6Zv/Jdzs4KjlZNrVUcmZggjF/IPHFSlrJeAxGREqxLJ3vhg+1Af3AAyJyQET+SUTi5qOKyIdFZL+I7O/vT72gyh8I8mv/+Bzf3Hc29cWnwMDYFHVJtolRlFxDRNjVXsO+04OzxjR3j05SlO/CU5q5hJJrwoH+42rFLDsZFxjgbcDeKPdYPrAd+JoxZhswAfxhvJuNMfcZY3YaY3bW1aU+nKu4II8er5+fvdK3iKUnT//4lMZflBXNrrYaBsanIvNnwLJgmj0lGf1gtanZEpijKjDLTjYIzF2E3WNhuoAuY8zz4Z8fwRIcx9jTXssLZ4YIBEOOPcfA+LRmkCkrml3tNQDsOz0YOdYz6qcxw+MU6iqKaKgs4pgG+pedjAqMiLiBm4Ef2MeMMT1Ap4jYverfABx3ch171tcwMR3kkIPT8LKpTYyiOMHq6lKa3cXs67gkMN2j/ozGX2w2Nbs1VTkDOJmm/DCwD7hSRLpE5IMi8hER+UjUZe8AnjDGzJ0Q9DvAt0TkMLAV+Aun1glwY1sNIrD31GDiixdBKNwmprZCXWTKykVEuLG9huc6hgiFjFVk6fXT5Mm8wFzT4uZU3ziT07GHosXj7MAEp/rGE1+oxMTJLLK7jTFNxpgCY0yrMebrxph7jTH3Rl3zoDHmrhj3HgzHVbYYY37JGOPoHFRPaSGbmt3sPT3gyOMP+6YJhoxaMMqKZ1dbDUMT07zWN8bg+BQzIUNjFjQk3dziJmTgRM/sOMxjR3voHPLFuQs+9b0j/P6/HXR6eSuWbIjBZAW719dw4PwwvumZtD/2wPg0oFX8ysrnxrZLcZhIinIWjLTe1GJV9EfHYR472sNHHnqJrz11Ou59J/vGea13jJBOx1wUKjBh9rTXEggaXjybfmNJ28QolwurqktprSrhuY5BuketKv5MtYmJprGymJqywsjI7c4hH3/wyCEATsdxgXn9AQbGp/AHQlwcjd0pWlkYFZgw162tpjDPxbOn0u8mW66RsYqSDexqq+H5M0NcGLF+752YopkqIsI1LW6OXvAyPRPiYw8fwGAl+JzunxsCtjgTdVzjMItDBSZMSWEe21Z7HInD9HjDApMFrgJFcZpd7TWM+AI8+WofhfkuqjJYZBnNpuZKXusd4//86DiHOkf4wru28Lor6hgYn2J0cn6V/5mBSwITT4SUhVGBiWLP+lqOXfQy4ptO6+N2j07iKS2gpNC5eRyKki3Y9TB7Tw1kZJJlPDa1uJkJGb657xzv3bWGOzY30V5XDkBH/3wLpWNgApdARXH+rOJRJXlUYKLYs74GY2YXiqWDbCg2U5TlosldwtqaUkImsz3I5mJX9F/TXMkf3bkRgLY6qwtVLAulo3+c1qpSrmyoUBfZIlGBiWJLq4eywry0u8mypdhMUZYL24rJZBfluayuKeUv37mZf3zvTooLLG/C6upS8l0S00I5MzDButoy1teXx7RwlMSowERRkOfihrYank1zwaU1kzx7/tAUxWnsdOVsS2y5+/rVs5IOCvJcrKkpnScgxhjODEzQVldGe105A+PTaXedXw6owMxhd3sNHQMTkRTLpeIPBBmcmFYLRrms2NVeQ1G+iw0N5ZleSkLa6srnuch6vVP4poO01ZbRXm+70dSKSRUVmDnsWV8LpK9tTJ/XqoFRgVEuJ+orinn2D1/P269tyfRSEtJeV865wQlmoprddgxYYrKutpz1dRUAnO7TTLJUUYGZw5UNFdSUFfJsmuIwtiWUTb5oRVkOasqLcLmyI4NsIdrryggEDZ3Dl7wWdopyW10ZLVUlFOa7OKUWTMqowMzB5RKuW1vN/jRV9EdqYNSCUZSspC2cqhxd0d/RP0FxgYvGymLyXEJbbVncin8lPiowMdixporzQz76xvxLfqxureJXlKymvW5+jOXMwARra8oiFlh7fblaMItABSYGO9ZWAfDyuaVbMT2jfiqK8ykvyl/yYymKkn48pYXUlhfSERXoPzMwESnCBCtO0znkwx9Ird3/5Y4KTAw2NbspynelxU3WPTqpAX5FyXLaassjFsz0TIjzQz7W1ZZFzq+vLydk4Nxg/Nb+ynxUYGJQmO/i2lYP+9NkwWgNjKJkN+31ZRGB6Rz2EQyZWQJju9G0oj81VGDisGNtFccuji7ZJO4e9WfFPAxFUeLTXlfOsC/A0MR0pIuy3UYGLAtHRGthUkUFJg4711QRCBoOdY4s+jECwRD941Ma4FeULMcWk47+8agamEsCU1KYR4unRC2YFFGBicP21Vagfylusr6xKUyWNfxTFGU+dkD/dP84ZwYmqC4rxFNaOO8atWBSQwUmDlVlhbTXlfHSEgSmJ4sm+imKEp/WqlIK81x09E/Q0T9BW5T1YtNeV05H/4SOT04BFZgF2LmmmpfPDy/6Fyoyk1yD/IqS1eS5hLW1pZzuH6cj3EV5Luvry5kMBJc0Pnl6JsS3nj/HmH/+gLOViArMAuxYW8WILxDxyaaKjkpWlNyhva6cw12j9I9Nsa4ulgUTf3ZMsvzZo8f49PeP8u8HLy76MXIJFZgF2LkmHIdZZD1M96if0sI8Kou1yFJRsp32unL6xqzmtG2187tAt9dbxxYb6H/4hfM89Nx5AI50LT55KJdQgVmAdbVl1JQVLjrQb9XAZM/IWEVR4jMrLTmGBVNTVointGBRgf6Xzg3xpz84yuuuqOV1V9RyuGt0SWvNFVRgFkBE2L6matGBfq3iV5Tcwc4kE7EmXc5FRKxMshQtmJ5RPx956GWaPSV89e7tbF3l4bXeMSanV37bGRWYBOxYU8WZgQkGxqdSvrdn1E9jpQb4FSUXsK2W1qqSyEjluWxoqOB4t5fpmVDM83OZmgnyWw+9hG9qhn98707cpQVsafUQMnC8O7EV0zPq5ze/uZ/7nj6d/EayCMcERkTuF5E+ETka5/w9InIw/HVURIIiUh11Pk9EDojIo06tMRnsOEyqVkwwZOgdm1ILRlFyhIriAhoqi1gXI/5ic9vGesb8M+xNcl7UT4/3cqhzhM+/awsbGqzBZVta3QAc6lxYYJ58tY87v/Jzfnq8l288ew5jci892kkL5kHg9ngnjTFfNMZsNcZsBT4FPGWMGYq65OPACQfXlxSbWtwU5rlSFpj+sSmCIaMZZIqSQ3z+nVv4n2/cEPf8L1xRS0VRPj8+3J3U4+07PUh5UT53bGqMHGuoLKa+oogjF2ILzEwwxF899grvf+BF6iuK+I0967gwMpmTjTYdExhjzNPAUMILLe4GHrZ/EJFW4C3APzmwtJQoLshjc6s7ZYG5NMlSBUZRcoVbr6rn2lWeuOeL8vN449UNPHG8l0AwsZtsX8cg16+rJj9v9lvtllYPh+Nkkn3om/v52pOnufv6Vfz7R/fw6zeuBkjaasomMh6DEZFSLEvnu1GHvwz8AZDwFRSRD4vIfhHZ39/f78gat6/2cOTCaNJ+V9AaGEVZqdy5uYnRyQB7Ty38ht/n9dPRP8GNbdXzzm1pddMxMDGv4PK13jGefLWfT9y2gb985xaKC/Joqy2jyV3Ms6cG07qP5SDjAgO8Ddhru8dE5K1AnzHmpWRuNsbcZ4zZaYzZWVdX58gCt6+uYnomxLGLyacWahW/oqxMXrch7CY7srCbbF+HJQg3ttXMO7e51Y0xcPSCd9bxHx/pRgTuvn5V5JiIsLu9lmdPD+Rcm5psEJi7iHKPAXuAXxSRs8C/AK8XkYcysTCb7eFA/8vnky+O6vH6Kcx3UVVa4NSyFEXJAEX5edyWhJvsuY4hKoryuabZPe/clhbr2JELs99Tfnykm+vWVFM/Z8THnvU1DPsCHO+eLUjZTkYFRkTcwM3AD+xjxphPGWNajTFrscTnv40x78nQEgErKNfiKeHl88nHYbpH/TRpkaWirEju2NTIiC/As6fju62eC8df8lzz3wNqyoto8ZTMKrg81TfGa73j3Lm5cd71e9bXAiR0y2UbTqYpPwzsA64UkS4R+aCIfEREPhJ12TuAJ4wxi2/us0xsW+3hQAqB/p7RSRp10JiirEhu2lBH+QLZZD2jfs4MTLCrfb57zGZLq3uWwPzocA8Ad2xumndtQ2Ux6+vL2buAoGUjTmaR3W2MaTLGFIQtkq8bY+41xtwbdc2Dxpi7FniMJ40xb3VqjamwfXUVF0f9keB9ImwLRlGUlUdxQR5v2FjP48d7YrrJnj8TP/5is6XVw/khHyO+aQB+crSbnWuqaIjzwXRPew0vnhliaiZ3OgBkQwwmJ7gUh0lsxYRChl6vn0YN8CvKiuXOzU2M+ALsi2FV7Ds9SGVxPhubKuPebxdcHrkwyun+cV7pGePOGNaLzZ71tUwGghxIIRacaVRgkuTqpkqK8l28nISbbHBimkDQqAWjKCuYmzfUUVaYFzObzIq/1MSMv9hsCgf6D3eNRlxtd8SIv9jc0FaDS+DZHIrDqMAkSWG+i80t7qQsGK2BUZSVj+Uma+AnR3voH7vUq7B7dJKzg76Y9S/RuEsKWFdbxuGuEX58tIcda6oWLGtwlxSwudWTU3EYFZgU2L6miqMXvAl9oFrFryiXB791cxtTM0F+85v78Qes94XnwvUvCwX4bTa3uHnm5AAnur2z2snE4xfW13CwcyRnJmKqwKTA9tUepoMhjl1cOBe9a9gWGI3BKMpK5ppmN1/+1a0c7Bzhk985RChkeO70EO6SAjY2xo+/2GxpdTMRbtu/UPzFZk97LcGQ4YUzyXbhyiwqMCmwbXU40J8gDtMxME5lcT615YXLsSxFUTLI7Zua+MM7ruLRw918+T9fY1/HIDesq8a1QPzFZkur1fds22oPzZ7EH0i3r6miKN/F3hxpG6OzfFPALrhMlMXR0T9BW125FlkqymXCb93URkf/OF/571MAvH/32qTuu6a5Ek9pAe/esSrxxVhxn+vWVvPMKWf6LqYbtWBSZNtqT8JA/+n+8ch0PEVRVj4iwv/5pc3sCte9JBN/ASgryueFP7ptVu+xRLzuilpe6x2PxHqzGRWYFNm+uoruUX/cF3fMH6DXOxVzpreiKCuXwnwX9713B//43p0L1r/Eui8Vb8dNG6ymvj9/LfvTlVVgUsQuuIznJjszYHW9UQtGUS4/KooLeOPVDY4+x1WNFdRXFPHUyex3k6nApEiigsvT/eMAtKsFoyiKA4gIN22o45mTAwSzvH2/CkyKJCq47OifIM8lrK4pXeaVKYpyuXDThjpGJwMcijMVM1tQgVkE29dUcfRi7ILL0/3jrK4upSg/LwMrUxTlcuB162sRgadfy243mQrMIti2ysP0TIgT3WPzznX0T9BWq+4xRVGco6qskC0tbhWYlYhdcHlgjpssGDJ0DEzQXq8BfkVRnOXmDXUc7Bxh1Je9bWNUYBZBo7uYxsrieZlkF4YnmZ4JqQWjKIrj3LShjpCBvaezN11ZBWaRbFvt4UDnbAvm9EA4g0wtGEVRHGbrKg8Vxfk89Wr2uslUYBbJttUeOocmZ7XpPt1npyirwCiK4iz5eS72tNfy9Ml+jMnOdGUVmEVix2EOdl5yk3UMTOApLaC6TJtcKoriPDdtqKN71M+p8IfbbEMFZpFsanaT7xIORrnJTvdpDzJFUZaPmzbUAvBUlmaTqcAskpLCPDY2Vc4K9J/WFGVFUZaR1qpS2uvKePpkdgb6VWCWwLbVHg51jhAMGUYnAwyMT2mAX1GUZeWmDXUcv+hlJhjK9FLmoQKzBLau8jAxHeRk3xgd/RrgVxRl+fm92zaw71OvJz8v+97OdeDYErhUcDlCYfjF1Tb9iqIsJ+6SgkwvIS4qMEtgbU0pntICDpwfpra8iHyXsLpam1wqiqKAusiWhIiwbZWHA+dH6OifYHVNKQVZaKYqiqJkAsfeDUXkfhHpE5Gjcc7fIyIHw19HRSQoItUiskpEfiYiJ0TkmIh83Kk1poNtq6s42TfOoa4Rjb8oiqJE4eTH7QeB2+OdNMZ80Riz1RizFfgU8JQxZgiYAf6nMWYjcCPwURG52sF1Loltqz0AdI/6Nf6iKIoShWMCY4x5GhhK8vK7gYfD93UbY14Ofz8GnABaHFlkGtjS6ol8rxaMoijKJTIeMBCRUixL57sxzq0FtgHPL3D/h0Vkv4js7+9f/mpWd0kB68O1LyowiqIol8i4wABvA/aG3WMRRKQcS3R+zxjjjXezMeY+Y8xOY8zOuro6h5cam22rLCumXV1kiqIoEbIhTfkuwu4xGxEpwBKXbxljvpeRVaXA+/esZW1tGZ5SbXKpKIpik1GBERE3cDPwnqhjAnwdOGGM+ZtMrS0Vrml2c02zO9PLUBRFySocExgReRi4BagVkS7gM0ABgDHm3vBl7wCeMMZMRN26B/gfwBERORg+9kfGmB87tVZFURQl/TgmMMaYu5O45kGsdOboY88A4syqFEVRlOUiG4L8iqIoygpEBUZRFEVxBBUYRVEUxRFUYBRFURRHUIFRx8THLAAACFdJREFUFEVRHEEFRlEURXEEMcZkeg1pQ0T6gXNzDruB0QTHon9O9H0tMLCEZcZaT7LXpLqXuT/b36+kvUR/v5T9LGUv8c7p79mlY/raJLfWRNc48dpcaYypSLzsRWCMWdFfwH2JjkX/nOh7YH+615PsNanuZYE9rJi9pGs/S9mL/p4t/Humr83KfW0SfV0OLrIfJnHshyl+n+71JHtNqnuZ+/MP41yzWLJhL8muIxFL2Uu8c/p7lh70tVn4eCZfmwVZUS6y5UBE9htjdmZ6HelgJe0FVtZ+VtJeYGXtZyXtBZzdz+VgwaSb+zK9gDSykvYCK2s/K2kvsLL2s5L2Ag7uRy0YRVEUxRHUglEURVEcQQVGURRFcYTLWmBE5H4R6RORo4u4d4eIHBGRUyLylfCgNPvc74jIqyJyTES+kN5Vx11P2vciIp8VkQsicjD8dWf6Vx53TY68NuHznxQRIyK16Vvxgutx4rX5nIgcDr8uT4hIc/pXHnM9TuzliyLySng/3xcRT/pXHndNTuzn3eG//ZCIOJ4MsJQ9xHm894nIyfDX+6KOL/h3FROn8p9z4Qu4CdgOHF3EvS8Au7Bm1/wEuCN8/FbgP4Gi8M/1ObyXzwKfXCmvTfjcKuBxrILc2lzdC1AZdc3vAvfm8F7eBOSHv/8r4K9y+fcM2AhcCTwJ7MzWPYTXt3bOsWqgI/xvVfj7qoX2u9DXZW3BGGOeBoaij4lIu4g8JiIvicjPReSqufeJSBPWH/g+Y/3PfxP4pfDp3wY+b4yZCj9Hn7O7sHBoLxnDwf38X+APgGXLbnFiL8YYb9SlZSzTfhzayxPGmJnwpc8Brc7u4hIO7eeEMebV5Vh/+PkWtYc4vBn4qTFmyBgzDPwUuH2x7xOXtcDE4T7gd4wxO4BPAn8f45oWoCvq567wMYANwOtE5HkReUpErnN0tQuz1L0AfCzsurhfRKqcW2pSLGk/IvKLwAVjzCGnF5oES35tROTPRaQT+HXgTx1cayLS8Xtm8xtYn44zSTr3kymS2UMsWoDOqJ/tfS1qv46NTM5FRKQc2A18J8q9WBTr0hjH7E+Q+Vim5Y3AdcC/iUhbWPWXjTTt5WvA58I/fw74EtYbwLKz1P2ISCnwaSx3TEZJ02uDMebTwKdF5FPAx4DPpHmpCUnXXsKP9WlgBvhWOteYCuncT6ZYaA8i8gHg4+Fj64Efi8g0cMYY8w7i72tR+1WBmY0LGDHGbI0+KCJ5wEvhH/8D64032oxvBS6Gv+8CvhcWlBdEJITVHK/fyYXHYMl7Mcb0Rt33j8CjTi44AUvdTzuwDjgU/qNrBV4WkeuNMT0Or30u6fg9i+bbwI/IgMCQpr2Eg8lvBd6w3B/G5pDu1yYTxNwDgDHmAeABABF5Eni/MeZs1CVdwC1RP7dixWq6WMx+nQ5AZfsXsJao4BjwLPDu8PcCXBvnvhexrBQ74HVn+PhHgD8Lf78By9yUHN1LU9Q1nwD+JZdfmznXnGWZgvwOvTZXRF3zO8AjObyX24HjQN1y/n45/XvGMgX5F7sH4gf5z2B5YarC31cns9+Y68rEC5otX8DDQDcQwFLoD2J9yn0MOBT+pf/TOPfuBI4Cp4GvcqkrQiHwUPjcy8Drc3gv/wwcAQ5jfWprWo69OLWfOdecZfmyyJx4bb4bPn4Yq3FhSw7v5RTWB7GD4a9lyYhzcD/vCD/WFNALPJ6NeyCGwISP/0b4NTkFfCDRfhf60lYxiqIoiiNoFpmiKIriCCowiqIoiiOowCiKoiiOoAKjKIqiOIIKjKIoiuIIKjDKikZExpf5+f5JRK5O02MFxeqWfFREfpioy7CIeETk/0vHcytKOtA0ZWVFIyLjxpjyND5evrnUmNFRotcuIt8AXjPG/PkC168FHjXGbFqO9SlKItSCUS47RKRORL4rIi+Gv/aEj18vIs+KyIHwv1eGj79fRL4jIj8EnhCRW0TkSRF5RKw5Jt+yZ2OEj+8Mfz8ebkh5SESeE5GG8PH28M8visifJWll7eNS085yEfkvEXlZrPkcbw9f83mgPWz1fDF87T3h5zksIv87jf+NipIQFRjlcuRvgf9rjLkOeBfwT+HjrwA3GWO2YXUn/ouoe3YB7zPGvD788zbg94CrgTZgT4znKQOeM8ZcCzwN/GbU8/9t+PkT9nMK98F6A1Y3BQA/8A5jzHas+UNfCgvcHwKnjTFbjTH3iMibgCuA64GtwA4RuSnR8ylKutBml8rlyG3A1VGdZitFpAJwA98QkSuwOsUWRN3zU2NM9MyNF4wxXQAichCrF9Qzc55nmksNQl8C3hj+fheXZml8G/jrOOssiXrsl7Bmc4DVC+ovwmIRwrJsGmLc/6bw14Hwz+VYgvN0nOdTlLSiAqNcjriAXcaYyeiDIvJ3wM+MMe8IxzOejDo9MecxpqK+DxL7bylgLgU5412zEJPGmK0i4sYSqo8CX8Ga/1IH7DDGBETkLFAc434B/tIY8w8pPq+ipAV1kSmXI09gzU8BQETstuZu4EL4+/c7+PzPYbnmAO5KdLExZhRrLPInRaQAa519YXG5FVgTvnQMqIi69XHgN8LzQRCRFhGpT9MeFCUhKjDKSqdURLqivn4f6816ZzjwfRxrxALAF4C/FJG9QJ6Da/o94PdF5AWgCRhNdIMx5gBWZ9y7sAZy7RSR/VjWzCvhawaBveG05i8aY57AcsHtE5EjwCPMFiBFcRRNU1aUZSY8XXPSGGNE5C7gbmPM2xPdpyi5hsZgFGX52QF8NZz5NUKGxlAritOoBaMoiqI4gsZgFEVRFEdQgVEURVEcQQVGURRFcQQVGEVRFMURVGAURVEUR/j/AbOfqlHcZd89AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot(skip_end=15)\n",
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='5', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/5 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='392970' class='' max='1040754', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      37.76% [392970/1040754 2:36:22<4:17:47 1.1668]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-60-491d2105eff6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1e-1\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# only one learning rate not slice(1e-3, 1e-5), because there is only one layer\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[1;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)\u001b[0m\n\u001b[0;32m     21\u001b[0m     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,\n\u001b[0;32m     22\u001b[0m                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))\n\u001b[1;32m---> 23\u001b[1;33m     \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr,  moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[0;32m    198\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mwd\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    199\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 200\u001b[1;33m         \u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    201\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m->\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[0;32m     99\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0myb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 101\u001b[1;33m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    102\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    103\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[1;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[0;32m     36\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_step_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m     \u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     37\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 38\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     39\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m def get_preds(model:nn.Module, dl:DataLoader, pbar:Optional[PBar]=None, cb_handler:Optional[CallbackHandler]=None,\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(5, 1e-1) # only one learning rate not slice(1e-3, 1e-5), because there is only one layer in tabular/collab data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We usually want the valid loss less than 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('dotprod')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interpretation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('dotprod');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Count the top\n",
    "g = rating_movie.groupby(title)['rating'].count()\n",
    "print(g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# See Top movies that's rated the most time (not good movies)\n",
    "top_movies = g.sort_values(ascending=False).index.values[:1000]\n",
    "top_movies[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interpreting Movie bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Movie Bias\n",
    "# We take the learner and look at the bias of the items(movies) in the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_bias = learn.bias(top_movies, is_item=True) # is_item=False would show the user bias\n",
    "movie_bias.shape # it should be 1000, as we try to see the bias for 1000 movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_ratings = rating_movie.groupby(title)['rating'].mean()\n",
    "movie_ratings = [(b, i, mean_ratings.loc[i]) for i,b in zip(top_movies,movie_bias)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "item0 = lambda o:o[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sort by the bias\n",
    "# this givs lowest numbers --> not a great movie\n",
    "sorted(movie_ratings, key=item0)[:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This gives the highest numbers --> great movie\n",
    "sorted(movie_ratings, key=lambda o: o[0], reverse=True)[:15]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interpreting Movie weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_w = learn.weight(top_movies, is_item=True)\n",
    "movie_w.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_pca = movie_w.pca(3) # we want to see the 3 highest PCA\n",
    "movie_pca.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fac0,fac1,fac2 = movie_pca.t()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let look at the features in PCA 1\n",
    "movie_comp = [(f, i) for f,i in zip(fac0, top_movies)] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We are not sure what this means\n",
    "# We can just see it's more like a classic movie\n",
    "sorted(movie_comp, key=itemgetter(0), reverse=True)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Same thing in another\n",
    "sorted(movie_comp, key=itemgetter(0))[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Features in PCA 2\n",
    "movie_comp = [(f, i) for f,i in zip(fac1, top_movies)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(movie_comp, key=itemgetter(0), reverse=True)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Graph Top 50 most popular movie by rating by the frequnecy\n",
    "idxs = np.random.choice(len(top_movies), 50, replace=False)\n",
    "idxs = list(range(50))\n",
    "X = fac0[idxs]\n",
    "Y = fac2[idxs]\n",
    "plt.figure(figsize=(15,15))\n",
    "plt.scatter(X, Y)\n",
    "for i, x, y in zip(top_movies[idxs], X, Y):\n",
    "    plt.text(x,y,i, color=np.random.rand(3)*0.7, fontsize=11)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}