Fast.ai/(Fast.ai) Text Classification V2 (Language Model).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *   # Quick access to NLP functionality\n",
    "from fastai.datasets import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "path = untar_data(URLs.IMDB_SAMPLE)\n",
    "df = pd.read_csv(path/'texts.csv')\n",
    "# Language model data\n",
    "data_lm = TextLMDataBunch.from_csv(path, 'texts.csv')\n",
    "# Classifier model data\n",
    "data_clas = TextClasDataBunch.from_csv(path, 'texts.csv', vocab=data_lm.train_ds.vocab, bs=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "      <th>is_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>negative</td>\n",
       "      <td>Un-bleeping-believable! Meg Ryan doesn't even ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>positive</td>\n",
       "      <td>This is a extremely well-made film. The acting...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>negative</td>\n",
       "      <td>Every once in a long while a movie will come a...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>positive</td>\n",
       "      <td>Name just says it all. I watched this movie wi...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>negative</td>\n",
       "      <td>This movie succeeds at being one of the most u...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      label                                               text  is_valid\n",
       "0  negative  Un-bleeping-believable! Meg Ryan doesn't even ...     False\n",
       "1  positive  This is a extremely well-made film. The acting...     False\n",
       "2  negative  Every once in a long while a movie will come a...     False\n",
       "3  positive  Name just says it all. I watched this movie wi...     False\n",
       "4  negative  This movie succeeds at being one of the most u...     False"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save\n",
    "data_lm.save('data_lm_export.pkl')\n",
    "data_clas.save('data_clas_export.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load\n",
    "data_lm = load_data(path, 'data_lm_export.pkl')\n",
    "data_clas = load_data(path, 'data_clas_export.pkl', bs=16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building a Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://s3.amazonaws.com/fast-ai-modelzoo/wt103-fwd\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.310412</td>\n",
       "      <td>3.884722</td>\n",
       "      <td>0.284473</td>\n",
       "      <td>00:27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Learn the model using transfer learning\n",
    "learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.5)\n",
    "learn.fit_one_cycle(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='24', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/24 00:00<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 50.00 MiB (GPU 0; 4.00 GiB total capacity; 2.85 GiB already allocated; 13.47 MiB free; 63.94 MiB cached)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-19-844d937719dd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munfreeze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1e-3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_encoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'ft_enc'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[0;32m    198\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mwd\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    199\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 200\u001b[1;33m         \u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    201\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m->\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[0;32m     99\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0myb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 101\u001b[1;33m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    102\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    103\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[1;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mxb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0myb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0myb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0myb\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 26\u001b[1;33m     \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     27\u001b[0m     \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_loss_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\container.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m     90\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 92\u001b[1;33m             \u001b[0minput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     93\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\learner.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    263\u001b[0m         \u001b[0mraw_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mmasks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    264\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 265\u001b[1;33m             \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    266\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m>\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msl\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    267\u001b[0m                 \u001b[0mmasks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpad_idx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\models\\awd_lstm.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, from_embeddings)\u001b[0m\n\u001b[0;32m    110\u001b[0m         \u001b[0mnew_hidden\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mraw_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    111\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhid_dp\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhidden_dps\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 112\u001b[1;33m             \u001b[0mraw_output\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_h\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrnn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mraw_output\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhidden\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0ml\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    113\u001b[0m             \u001b[0mnew_hidden\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_h\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    114\u001b[0m             \u001b[0mraw_outputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mraw_output\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\models\\awd_lstm.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m     47\u001b[0m             \u001b[1;31m#To avoid the warning that comes because the weights aren't flattened.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     48\u001b[0m             \u001b[0mwarnings\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msimplefilter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"ignore\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 49\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     50\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     51\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\rnn.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, hx)\u001b[0m\n\u001b[0;32m    562\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward_packed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    563\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 564\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    565\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    566\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\rnn.py\u001b[0m in \u001b[0;36mforward_tensor\u001b[1;34m(self, input, hx)\u001b[0m\n\u001b[0;32m    541\u001b[0m         \u001b[0munsorted_indices\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    542\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 543\u001b[1;33m         \u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_batch_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msorted_indices\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    544\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    545\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpermute_hidden\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munsorted_indices\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\rnn.py\u001b[0m in \u001b[0;36mforward_impl\u001b[1;34m(self, input, hx, batch_sizes, max_batch_size, sorted_indices)\u001b[0m\n\u001b[0;32m    524\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mbatch_sizes\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    525\u001b[0m             result = _VF.lstm(input, hx, self._get_flat_weights(), self.bias, self.num_layers,\n\u001b[1;32m--> 526\u001b[1;33m                               self.dropout, self.training, self.bidirectional, self.batch_first)\n\u001b[0m\u001b[0;32m    527\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    528\u001b[0m             result = _VF.lstm(input, batch_sizes, hx, self._get_flat_weights(), self.bias,\n",
      "\u001b[1;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 50.00 MiB (GPU 0; 4.00 GiB total capacity; 2.85 GiB already allocated; 13.47 MiB free; 63.94 MiB cached)"
     ]
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit(1, 1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predict\n",
    "learn.predict(\"This is a review about\", n_words=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save ENCODER\n",
    "learn.save_encoder('ft_enc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building a classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5)\n",
    "learn.load_encoder('ft_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit_one_cycle(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze_to(-2)\n",
    "learn.fit_one_cycle(1, slice(5e-3/2., 5e-3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(1, slice(2e-3/100, 2e-3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predict\n",
    "learn.predict(\"This was a great movie!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Classification Interpretation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.cm as cm\n",
    "\n",
    "txt_ci = TextClassificationInterpretation.from_learner(learn) # TextClassificationInterpretation can be used ONLY once we train a text classification model.\n",
    "test_text = \"Zombiegeddon was perhaps the GREATEST movie i have ever seen!\"\n",
    "txt_ci.show_intrinsic_attention(test_text,cmap=cm.Purples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_ci.intrinsic_attention(test_text)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt_ci.show_top_losses(5) # If max_len=None, it will display all tokens."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using DataBLock API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# So another approach to doing this is to just decide:\n",
    "\n",
    "#### What kind of list you're creating (i.e. what's your independent variable)? So in this case, my independent variable is text.\n",
    "#### What is it coming from? A CSV.\n",
    "### How do you want to split it into validation versus training? So in this case, column number two was the is_valid flag.\n",
    "### How do you want to label it? With positive or negative sentiment, for example. So column zero had that.\n",
    "### Then turn that into a data bunch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = (TextList.from_csv(path, 'texts.csv', cols='text')\n",
    "                .split_from_df(col=2)\n",
    "                .label_from_df(cols=0)\n",
    "                .databunch())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Language Model --> Guessing the next word of the movie review!!\n",
    "### Accuracy: 0.3 or more is usually pretty good --> Predict the next word in 1/3 of the time!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://s3.amazonaws.com/fast-ai-nlp/imdb\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[WindowsPath('C:/Users/bokhy/.fastai/data/imdb/imdb.vocab'),\n",
       " WindowsPath('C:/Users/bokhy/.fastai/data/imdb/README'),\n",
       " WindowsPath('C:/Users/bokhy/.fastai/data/imdb/test'),\n",
       " WindowsPath('C:/Users/bokhy/.fastai/data/imdb/tmp_clas'),\n",
       " WindowsPath('C:/Users/bokhy/.fastai/data/imdb/tmp_lm'),\n",
       " WindowsPath('C:/Users/bokhy/.fastai/data/imdb/train'),\n",
       " WindowsPath('C:/Users/bokhy/.fastai/data/imdb/unsup')]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = untar_data(URLs.IMDB)\n",
    "path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[WindowsPath('C:/Users/bokhy/.fastai/data/imdb/train/labeledBow.feat'),\n",
       " WindowsPath('C:/Users/bokhy/.fastai/data/imdb/train/neg'),\n",
       " WindowsPath('C:/Users/bokhy/.fastai/data/imdb/train/pos'),\n",
       " WindowsPath('C:/Users/bokhy/.fastai/data/imdb/train/unsupBow.feat')]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(path/'train').ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Because the English of the reviews left by people on IMDB isn't the same as the English of wikipedia, \n",
    "# we'll need to adjust the parameters of our model by a little bit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# This is a text databunch model (sames as above)\n",
    "data_lm = (TextList.from_folder(path)\n",
    "           #Inputs: all the text files in path\n",
    "            .filter_by_folder(include=['train', 'test', 'unsup']) \n",
    "           #We may have other temp folders that contain text files so we only keep what's in train and test\n",
    "            .split_by_rand_pct(0.1)\n",
    "           #We randomly split and keep 10% (10,000 reviews) for validation\n",
    "            .label_for_lm()           \n",
    "           #We want to do a language model so we label accordingly\n",
    "            .databunch(bs=bs))\n",
    "data_lm.save('data_lm.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>idx</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>later , by which time i did not care . xxmaj the character we should really care about is a very cocky , overconfident xxmaj ashton xxmaj kutcher . xxmaj the problem is he comes off as kid who thinks he 's better than anyone else around him and shows no signs of a cluttered closet . xxmaj his only obstacle appears to be winning over xxmaj costner . xxmaj</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>the xxmaj dark \" that i actually remember seeing in the theater . \\n \\n  xxmaj the remaining 2 were equally as bad . \" xxmaj creatures from the xxmaj abyss \" being probably the worst of the bunch . xxmaj since they were all bad that is n't saying much . xxmaj but its stupidity , bad acting , terrible effects and retarded storyline actually made this one</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>almost 500 movies this is the only non - short i 've given a 1 / 10 . \\n \\n  i have n't seen any other low - budget xxmaj asian warrior flicks , so i guess there 's even worse things out there ! xxmaj scary ... \\n \\n  xxup :p xxbos so if a guy meets you and he says ' i want you to look</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>enough to xxup not dislike a film solely due to its lack of stars or professional look ? xxmaj well , if you said yes then you should have a mind open enough to handle this one . xxmaj this film is an almost 100 % accurate dramatization of xxmaj dahmer 's adult life and subsequent murder spree , and is styled as an autobiography . xxmaj it is n't</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>never been ' average ' . xxmaj when xxmaj madonna performs , she gives 120 % - everything is bigger , better , theatrical and meaningful . xxmaj she takes inspirations from xxmaj east , xxmaj west , and makes then unique . xxmaj everything is given a deeper meaning , and xxmaj the xxmaj confessions xxmaj tour is not different . xxmaj from the very beginning , you find</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# This it to load the above databunch (you can skip above one for upcoming models)\n",
    "data_lm = load_data(path, 'data_lm.pkl', bs=bs)\n",
    "data_lm.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Learn the model using transfer learning\n",
    "learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='99' class='' max='19305', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.51% [99/19305 00:38<2:04:57 12.2584]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEGCAYAAABYV4NmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxddZ3/8dcnSZO0adqmTbqmbdpSCqVAoYECRVZl/7GI+APEH8so6iiDOuoM4wwwIIiKiqgDggOKDqIijoAoWylbWyClLF3olu5LkiZpm327n98f9wRCSNK0vedueT8fj/vIued8zz2fb25yP/d7vt/zPebuiIiIxFpGogMQEZH0pAQjIiKhUIIREZFQKMGIiEgolGBERCQUWYkOIFYKCwu9pKQk0WGIiKSUJUuW7HT3ojBeO20STElJCWVlZYkOQ0QkpZjZxrBeW6fIREQkFKEmGDPbYGbvmtlbZvaR5oVF3W1ma83sHTM7usu2K81sTfC4Msw4RUQk9uJxiuxUd9/Zy7azgenBYy5wDzDXzEYCNwGlgANLzOxxd6+NQ7wiIhIDiT5FdgHwkEctBkaY2TjgTOBZd68JksqzwFmJDFRERPZN2AnGgWfMbImZXdvD9gnA5i7PtwTrelv/IWZ2rZmVmVlZVVVVDMMWEZEDFXaCmefuRxM9FfZlMzup23brYR/vY/2HV7jf5+6l7l5aVBTKKDsREdlPoSYYd98W/KwE/gwc263IFmBil+fFwLY+1ouISIoILcGYWZ6Z5XcuA2cAy7oVexz4f8FosuOA3e6+HXgaOMPMCsysINj36bBiFRFJVY8u2cLvXt+U6DB6FOYosjHAn82s8zgPu/vfzeyLAO5+L/AUcA6wFmgErg621ZjZrcAbwWvd4u41IcYqIpKS/li2mYg7lx07KdGhfERoCcbdy4Eje1h/b5dlB77cy/4PAA+EFZ+ISDqoqmvh0HHDEh1GjxI9TFlERA5AVV0LRfk5iQ6jR0owIiIpqqm1g7qWdiUYERGJrcq6ZgBGK8GIiEgsVdW1AKgFIyIisVUZJJjR+bkJjqRnSjAiIimqck/0FJlaMCIiElNV9S1kZhij8rITHUqPlGBERFJU5Z4WCodmk5HR0/SNiacEIyKSoqrqk/caGFCCERFJWZV7WpK2gx+UYEREUlZlXUvSXgMDSjAiIimpI+LUNOgUmYiIxFh1fQsRT96r+EEJRkQkJVUm+VX8oAQjIpKSPpgmRp38IiISQ8k+0SUowYiIpKRkn+gSlGBERFJSZV0Lw3KzyB2UmehQeqUEIyKSgqrqWhg9LHn7X0AJRkQkJVXWtVA0NHlPj4ESjIhISqqsa2b0MCUYERGJIXenSi0YMLNMM1tqZk/2sG2ymT1vZu+Y2QIzK+6yrcPM3goej4cdp4hIqqhraae5LZL0LZisOBzjemAlMKyHbXcCD7n7r83sNOC7wGeDbU3uPjsO8YmIpJTKPcl9q+ROobZgghbJucAveykyE3g+WH4BuCDMeERE0kEqXAMD4Z8iuwv4FhDpZfvbwMXB8kVAvpmNCp7nmlmZmS02swtDjlNEJGWkwlX8EGKCMbPzgEp3X9JHsW8AJ5vZUuBkYCvQHmyb5O6lwOXAXWY2rYdjXBskobKqqqoY10BEJDmpBQPzgPPNbAPwCHCamf22awF33+bun3T3o4BvB+t2d24LfpYDC4Cjuh/A3e9z91J3Ly0qKgqxKiIiyaOqroXsrAyGDx6U6FD6FFqCcfcb3L3Y3UuAS4H57n5F1zJmVmhmnTHcADwQrC8ws5zOMkST1YqwYhURSSWdF1maWaJD6VPcr4Mxs1vM7Pzg6SnAKjNbDYwBbgvWHwqUmdnbRDv/73B3JRgREaItmGQ/PQbxGaaMuy8gepoLd7+xy/pHgUd7KL8QODwesYmIpJrKumZKRuUlOoy90pX8IiIpJjrRZfK3YJRgRERSSGt7hNrGNoqGJvdFlqAEIyKSUqrqg6v41YIREZFYev8amCSf6BKUYEREUkrlnuAqfrVgREQklt4/RZbkE12CEoyISEqp3NOCGYwamp3oUPZKCUZEJIVU1rUwckg2gzKT/+M7+SMUEZH3pcpV/KAEIyKSUqrqmpVgREQk9rbtbmbc8OTv4AclGBGRlNHU2kFVXQuTRg5JdCj9ogQjIpIiNtc2AjBRCUZERGJpc40SjIiIhGBTkGB0ikxERGJqc00TgwdlMiov+S+yBCUYEZGUsammkUkjhyT9rZI7KcGIiKSILbWNTBw5ONFh9JsSjIhICnB3NtU0pkwHPyjBiIikhJqGVhpbO5hYoAQjIiIxlGojyEAJRkQkJWyubQJg0iglGBERiaHOiyyLC9TJ/z4zyzSzpWb2ZA/bJpvZ82b2jpktMLPiLtuuNLM1wePKsOMUEUlmm6obKRyaw5DsrESH0m/xaMFcD6zsZdudwEPufgRwC/BdADMbCdwEzAWOBW4ys4I4xCoikpQ2p9gQZQg5wQQtknOBX/ZSZCbwfLD8AnBBsHwm8Ky717h7LfAscFaYsYqIJLPOiyxTSdgtmLuAbwGRXra/DVwcLF8E5JvZKGACsLlLuS3Bug8xs2vNrMzMyqqqqmIXtYhIEmnriLB9d3NKDVGGEBOMmZ0HVLr7kj6KfQM42cyWAicDW4F2oKd5EPwjK9zvc/dSdy8tKiqKRdgiIkln+65mOiKuFkwX84DzzWwD8Ahwmpn9tmsBd9/m7p9096OAbwfrdhNtsUzsUrQY2BZirCIiSSvV7gPTKbQE4+43uHuxu5cAlwLz3f2KrmXMrNDMOmO4AXggWH4aOMPMCoLO/TOCdSIiA86m9+8Do07+PpnZLWZ2fvD0FGCVma0GxgC3Abh7DXAr8EbwuCVYJyIy4GyuaSQrwxg3PLUSTFwGVLv7AmBBsHxjl/WPAo/2ss8DfNCiEREZsDbVNDKhYDCZGakxTX8nXckvIpLkNtc2pVwHPyjBiIgkvc01jRSn2BBlUIIREUlq9S3t1DS0qgUjIiKxtTkFp+nvpAQjIpLEUnWIMijBiIgkNbVgREQkFJtrGsnPyWL44EGJDmWfKcGIiCSxzbVNTBw5BLPUugYGlGBERJLapprUuw9MJyUYEZEkFYk4m1PwPjCdlGBERJLU1l1NtLRHmFo0NNGh7BclGBGRJFW+swGAqYV5CY5k/yjBiIgkqfKqegC1YEREJLbKqxrIz82icGh2okPZL0owIiJJal1VPVOLhqbkEGVQghERSVrlVQ1MK0rN/hdQghERSUoNLe3s2NPMtBTtfwElGBGRpLQ+xUeQgRKMiEhSWpfiI8hACUZEJCmVVzVgBpNHpeZV/KAEIyKSlMp3NlBcMJjcQZmJDmW/KcGIiCShdZX1Kd3BD0owIiJJJxJx1u9sYGqhEkyfzCzTzJaa2ZM9bJtkZi8E298xs3OC9SVm1mRmbwWPe8OOU0QkWezY00xTWwdTU/gaGICsOBzjemAlMKyHbf8O/MHd7zGzmcBTQEmwbZ27z45DfCIiSaW8KhiinOIJJtQWjJkVA+cCv+yliPNB4hkObAszHhGRVFC+MzpEeUD0wZjZNDPLCZZPMbN/MrMR/dj1LuBbQKSX7TcDV5jZFqKtl+u6bJsSnDp70cw+1ktc15pZmZmVVVVV9acqIiJJr7yqgbzsTEbn5yQ6lAPS3xbMn4AOMzsI+G9gCvBwXzuY2XlApbsv6aPYZcCv3L0YOAf4jZllANuBSe5+FPB14GEz+8gpNne/z91L3b20qKion1UREUluqT7JZaf+JpiIu7cDFwF3ufvXgHF72WcecL6ZbQAeAU4zs992K/MPwB8A3H0RkAsUunuLu1cH65cA64CD+xmriEhKS/VJLjv1N8G0mdllwJVA52iwQX3t4O43uHuxu5cAlwLz3f2KbsU2AacDmNmhRBNMlZkVmVlmsH4qMB0o72esIiIpq6m1g627mlJ6iphO/U0wVwPHA7e5+3ozmwJ0b430i5ndYmbnB0//Gfi8mb0N/A64yt0dOAl4J1j/KPBFd6/Zn+OJiKSS9ye5TIMWTL+GKbv7CuCfAMysAMh39zv6exB3XwAsCJZv7Pa683oo/yei/T4iIgNK5wiyVL/IEvo/imyBmQ0zs5HA28CDZvajcEMTERl4Oq+BmZLC0/R36u8psuHuvgf4JPCgu88BPh5eWCIiA9O6qnomjBjM4OzUneSyU38TTJaZjQM+zQed/CIiEmPlVQ1p0f8C/U8wtwBPE52+5Y1gZNea8MISERl43J3yqtSfRblTfzv5/wj8scvzcuDisIISERmIKva00NDakRbXwED/O/mLzezPZlZpZhVm9qdgnjEREYmRVRV1AEwfk5/gSGKjv6fIHgQeB8YDE4AngnUiIhIjq3dEE8zBAyzBFLn7g+7eHjx+BWjyLxGRGFpVUUdRfg4j87ITHUpM9DfB7DSzK4Kbh2Wa2RVAdZiBiYgMNGsq6piRJq0X6H+CuYboEOUdRGc6/hTR6WNERCQGIhFndUU908ekxwgy6GeCcfdN7n6+uxe5+2h3v5DoRZciIhIDW2qbaGrrGJAtmJ58PWZRiIgMcJ0jyA4eqwQDkNp3whERSSKrO4cojx5gp8h64TGLQkRkgFtdUceEEYPJz+3zVlsppc8r+c2sjp4TiQGDQ4lIRGQAWrWjjoPTqIMf9pJg3D19TgaKiCSp9o4I5VUNnDwjvS4vPJBTZCIiEgMbqhtp7Yhw8Oj0+k6vBCMikmCdHfwz0mgEGSjBiIgk3KoddZjBQWk0ggyUYEREEm5NZR0lo/LIHZT6d7HsSglGRCTBVu2oS6vrXzopwYiIJFBzWwcbqhvTrv8FlGBERBKqvKqBjoinzT1gugo9wQTT+y81syd72DbJzF4Itr9jZud02XaDma01s1VmdmbYcYqIJMKayvQcQQZ7udAyRq4HVgLDetj278Af3P0eM5sJPAWUBMuXAocRvYvmc2Z2sLt3xCFeEZG4WbWjjqwMo2RUXqJDiblQWzBmVgycC/yylyLOB4lnOLAtWL4AeMTdW9x9PbAWODbMWEVEEmF1RR1Ti/LIzkq/Houwa3QX8C0g0sv2m4ErzGwL0dbLdcH6CcDmLuW2BOs+xMyuNbMyMyurqqqKWdAiIvGyqqIuLftfIMQEY2bnAZXuvqSPYpcBv3L3YuAc4DdmlkHPtwL4yKSb7n6fu5e6e2lRUXrN4SMi6a+xtZ3NNU1pdZOxrsLsg5kHnB903OcCw8zst+5+RZcy/wCcBeDui8wsFygk2mKZ2KVcMR+cPhMRSQvLt+0B0rODH0Jswbj7De5e7O4lRDvs53dLLgCbgNMBzOxQoomoCngcuNTMcsxsCjAdeD2sWEVEEmHxumrM4JiSkYkOJRTxGEX2IWZ2C1Dm7o8D/wzcb2ZfI3oK7Cp3d2C5mf0BWAG0A1/WCDIRSTeLyqs5ZOwwCvKyEx1KKOKSYNx9AbAgWL6xy/oVRE+l9bTPbcBtcQhPRCTuWto7WLKxls/MnZzoUEKTfuPiRERSwNJNu2hpj3D8tFGJDiU0SjAiIgmwaF01GQbHTknP/hdQghERSYhF5dUcNn44wwcPSnQooVGCERGJs+a2Dt7atIvjpqZv6wWUYERE4u7NjbW0dqR3/wsowYiIxN2i8moyMyxtr3/ppAQjIhJni9ZVM2vCcPJz07f/BZRgRETiqrG1nbe37OL4qel9egyUYERE4qpsQy1tHZ72/S+gBCMiEleLyqvJyjBKJxckOpTQKcGIiMTRonXVHFE8nLycuE8FGXdKMCIicVLf0s67W3cPiNNjoAQjIhI3b2yooSPiHD+1MNGhxIUSjIhInLxWXsOgTOPoySMSHUpcKMGIiMTJ4vJqjigewZDs9O9/ASUYEZG4aAj6X9J9/rGulGBEROKgbGMtHRFn7pSB0cEPSjAiInHxWjD/2JwBcP1LJyUYEZE4eG19zYC5/qWTEoyISMgaW9t5e/OuAXV6DJRgRERC9+bGXbRHnLkDqIMflGBEREK3OOh/GQjzj3UV+slAM8sEyoCt7n5et20/Bk4Nng4BRrv7iGBbB/BusG2Tu58fdqwiImF4bX01s8YPS/v7v3QXj96m64GVwLDuG9z9a53LZnYdcFSXzU3uPjv88EREwtPU2sFbm3dxzbwpiQ4l7kI9RWZmxcC5wC/7Ufwy4HdhxiMiEm9LN0Xv/zLQ+l8g/D6Yu4BvAZG+CpnZZGAKML/L6lwzKzOzxWZ2YYgxioiEZvH6GjIMSkuUYGLGzM4DKt19ST+KXwo86u4dXdZNcvdS4HLgLjOb1sMxrg2SUFlVVVVsAhcRiaHF5dUcNn44wwZY/wuE24KZB5xvZhuAR4DTzOy3vZS9lG6nx9x9W/CzHFjAh/tnOsvc5+6l7l5aVFQUw9BFRA5cc1u0/2XulIHXeoEQE4y73+Duxe5eQjSBzHf3K7qXM7MZQAGwqMu6AjPLCZYLiSarFWHFKiIShqWbdtHaHuG4qQPrAstOcb8OxsxuMbOuQ44vAx5xd++y7lCgzMzeBl4A7nD30BLMrsZWGlraw3p5ERmgXlpTRWaGccwAbcHEZVIcd19A9DQX7n5jt20391B+IXB4HEJjU3Ujp/5wAd+5cBaXHTspHocUkQHi2RUVHFsykuGDB17/C+hKfiaOHExxwWD+tmxHokMRkTSyfmcDayvrOeOwMYkOJWEGfIIxM86aNZaFa3eyu7Et0eGISJp4dkX0S+snZirBDGhnzxpHe8R5bmVFokMRkTTx7IoKDh03jOKCIYkOJWGUYIAji4czbniuTpOJSEzsrG+hbGPtgG69gBIMED1NduZhY3lpTRX1Gk0mIgdo/spK3OEMJRgBOHvWWFrbI7zwXmWiQxGRFPfMigrGD8/lsPEfmeN3QFGCCZSWjKRwaDZ/12kyETkATa0dvLK2ik/MHIOZJTqchFKCCWRmGGccNpYXVlXS3Nax9x1ERHrw8poqmtsifGLm2ESHknBKMF2cPWssja0dvLhaE2eKyP55dkUF+blZA3J6/u6UYLo4buoohg8epNNkIrJfOiLO8+9VcuqM0QzK1MdrXKaKSRWDMjP4xMwxPL18B63tEbKzev4D2VzTSPnOBuqb26lrbqO+pZ2i/BzOnjWu131EJP0t2VhLTUPrgL56vyslmG7OnjWWR5ds4f6XyymdXMCoodmMGJLN6oo6XnivkvnvVbKuqqHHfe8Y/h6f/9hULj12IkOy9atNRZtrGnnsza3kDMogPzeL/NxB5GZl0NDaTl1z9NERcS4pLWbc8MGJDleSzO/f2Ex2VgYnH6zbh4ASzEecOL2QUXnZ/ODpVR/Zlp2ZwdypI/nM3MkcXhy9gVB+bhZDc7N4c2Mt/7VgHbc8uYKfzl/Dl089iM99bGoCaiD7a1djK5/979fYUN2417L3vriOr358OlfPm6JTIQLAsq27eWzpFj7/sankD8Cbi/VECaabnKxMXvjmKWypaaKmoZWaxlZq6lsYN2IwJx5USF5Oz7+yU2aM5pQZoynbUMNPnl/Dd/66koIh2Vw8pzjONYhauqmWZVt3M3b4YMaPyKV4xBCGDc4a8MMme9PeEeErDy9l664mHv3i8cwcPyxosbTR1BohLyeT/OALReWeFm5+Yjm3P/Uejy7ZwncuPJxj+5iOfVN1I6sq6qiub6G6oZXq+lYGZ2cwtXAo00YPZWpRXp93O3R32iOuRJbE3J3b/rqSEYMH8eVTD0p0OEnDPnwbltRVWlrqZWVliQ4DiHb0XXb/YpZt3c2T153I1KKhB/yaW3c10dYeYULB4D4/aFbtqOMHT6/qcV61scNy+dIp07j02InkZGUecEzp5ObHl/OrhRv4waeO4JLSiXst7+48u6KC/3xiBVt3NXHh7PHccM6hjBmW+36Z5rYO7n5+Db94qZyOyAf/Z0OyM2lpj3xo3aSRQzh1RhGnHTqG46aOJDszg6Wbd/HUO9v527Id7Kxv4VtnHcLVJ5SQkaEvCcnmuRUVfO6hMv7z/MO48oSSRIezT8xsSXB7+ti/thJMOLbvbuLsn7xMccFg/vSlEw7oA728qp5z736FprYOMjOM8SNyKRmVx/jhgynKz6EoP4fCoTnMf6+Sx5ZuYWh2Fl88ZRoXHTWBqroWtu1qYuuuJp5ZUcHr62sYPzyX606fzqfmFOtbMfC71zdxw2Pv8rkTp/Dv583cp30bW9v5+Qtruf+l9QzKNK47fTrXzJvCko21/Nuf32X9zgYumVPMFcdNpjA/h1F52eQOyqStI8KmmkbWVdZTvrOBsg01vLJ2J81tEYZkZ5Kfm0XFnhayMzP42PRC2iPOi6urmHfQKO685Ej1/ySRto4IZ/74JTB4+qsnpdz/lBJMPyRbggF4ZvkOrv3Nkv364OrU3hHh4nsXsWFnA/969iFsrW1iY00jm6ob2L67meqG1ve/CedkZXDVCSV88eRpFORlf+S13J1X11Zz5zOreGvzLsYMy+HgMfkUFwyhuGAwUwrzOO2Q0eQOGjitm/nvVXDtQ0uYd1AhD1x1DJn72TrYWN3ArU+u5LmVFYzOz6GyroVJI4dw+0WHc+L0wn69RnNbB4vWVfPcygpqGlr5xMwxfHzmGIblDsLdeeSNzdzyxAoGZRq3XDCLsw8fq5ZoEvjVq+u5+YkV/PeVpZx+aOqNHlOC6YdkTDAAN/5lGQ8t2siDVx/DqTNG7/P+dz23mrueW8PPLj+K844Y/5HtHRGntrGVqrqW91sye+PuvLCqkj8t2crm2ka21Eb7mwAKh+ZwzYklXHHc5D77BVJdTUMr3/nrCh57cyszxuTzhy8eH5O7Di5YVcldz61h7tSRfPX0gxmcHdsEsH5nA1/7/Vu8tXkXOVkZzJlcwHFTR3Hc1FHMmVyw3wlS9s/uxjZOvvMFZo4bxv98bm5K9nEqwfRDsiaY5rYOLvz5q1TWtfC//ziPSaP6f2+Itzbv4uJ7FnL+keP58f+dHWKU0VM9Szft4hcvlfPS6iryc7K4/LhJzBo/nJF52RQMyaZwaDaju/QxpCJ3509vbuW2v66grrmdL548ja+cdlBKtdraOyLMf6+SxeU1LC6vZuWOPbhDUX4O/+eI8Vx01ARmTRiWkh92qeY/n4j23T153YkcNn54osPZL0ow/ZCsCQZgXVU9F9+zkGG5g3j0S8czOn/vH9KNre2ce/crtLR18LevnhTXe3ov27qbe15cx1Pvbqf7n8c5h4/lexcfkbLDMG/6yzJ+vWgjcyYXcPtFhzNjbH6iQzpguxvbeGXtTh5/eysvvFdFa0eEaUV5/PDTs5k9cUSiw0tbb26q5eJ7FnL5sZO47aLDEx3OflOC6YdkTjAQHTb8mV++xqSRQ/j9Fz44HROJOL8v28z9L5eTn5NFSWEek0flsaaijr8v38HDnzuO46eNSkjMuxpbqaxrobahldrGVpZt3cM9L65j0sgh3HPF0RwyNrWmIt+6q4mTv/8CFx9dzHc/eXhajsba3djGU8u287P5a2ntiPDkdSd+aGSbxEZLewfn3v0KjS3tPP21k1L2CxeEm2BSa7hDCjtqUgG/+Owc1lXV8w+/eoOm1g7e3bKbi+5ZyA2PvRtcYzGIJRtr+en8Nfxt2Q4+/7GpCUsuACOGZHPwmHzmTh3FWbPG8Y0zZ/Dw5+ZS39LOhT9/lcfe3IK7s6uxlbWV9by+vobaoC8nLKt21FFZ17xf+/73y+sB+KePT0/L5AIwfMggLjt2Eg9efQwNLe184TdLaGn/6Ozgr6+voWxDDZFIenzBjLefPr+WtZX13P7Jw1M6uYRNLZg4++s72/nK795kSmEe63c2MCovm38751AuOmrC++fMW9o7qNjdQnHB4KT8IKysa+YrDy/l9fU1ZGUY7V0+pAqHZvOLz5YyZ3LBR/Zbv7OBmobWHrftTSTi3P7USn75SjRJTBo5hNKSAo4pGck5s8YxfEjf/+S7Gls54Y75nHXYWH4Ucn9Wsvj7su188bdv8unSYr538RGYGbsaW7nxL8t5/O1tABQXDOaC2dF+m4NGp/7pwnhYtnU3F/z8VS46agJ3XnJkosM5YCl9iszMMoEyYKu7n9dt24+BU4OnQ4DR7j4i2HYl8O/Btu+4+6/7Ok6qJBiAh1/bxM2PL+fyuZP42icOjmv/Sqy0d0R4aNFGdta3MGpoDoVDo9d33P7USrbvauZ7nzqci46KzmLQ3NbBz19Yy70vrsMdHvvHEziiuP99Ay3tHfzzH97myXe2c/ncSUwZlUfZxhrKNtRS3dDKyLxs/uWsGVwyZ2KvCfnu59fwo2dX8/RXT0qLfpf++tEzq7h7/lpuveAwxo8YzL8+9i61Da1cd9p0Jo0azJ+XbuOVNVVEPDqCsHBoNqOGZjMqL4cJBYOZOW4Yh40fRsmovKT8shNvbR0RLvjZq1TVt/Dc107e6xebVJDqCebrQCkwrHuC6VbuOuAod7/GzEYSTUqlgANLgDnuXtvb/qmUYCD6AZ2VYhdk9UdtQytf+p8lLC6v4UunTOO4qaO48S/L2FjdyIWzx/P6+hpyBmXy5HUn9jrtTle7G9u49jdlvLa+hhvOPoRrT5r6fkvP3Xl3625ufXIFb2yoZfbEEdx6wSwOL/7waJ6m1g7mfW8+syeO4IGrjgml3skqEnE+/1AZL6yqJOJwyNh87rzkSGZN+OB3VFnXzF/f2c7qijp21rdSXd/CzvpWtu9uoq0j+vmQl53JcVNH8e1zD43JzBSpyN353t9Xce+L6/jFZ+dw5mHpcUOxlE0wZlYM/Bq4Dfj6XhLMQuAmd3/WzC4DTnH3LwTbfgEscPff9bZ/qiWYdNbaHuGmx5fzu9c3ATClMI/vXDiLeQcVsri8msvuX8z/LZ3IHRcf0etr1DS08sJ7ldz74jo2VDdw5yVHcsHsCT2WdXf+vHQrtz/1HtUNLVx5fAnfOmvG+zNa/2bRBv7jL8v5wxeO73POsHS1p7mN6x5eyhHFw/nKaQf1++LMlvYO1lTUs3zbbpZt3cP/vrWVlvYI158+nWtPmppyV6wfiPqWdv7l0Xf467vb+XRpMd//VOqfGuuUygnmUeC7QD7wjd4SjJlNBhYDxWfiWBIAAA1+SURBVO7eYWbfAHLd/TvB9v8Amtz9zm77XQtcCzBp0qQ5GzduDK0usm/cnd+/sZmaxlaumTflQ9eZfP/v7/FfC9Zx7xVHc9asce+vr9zTzP++tZXnVlRStrGGiMO44bn88JIjOeGgvV8Nv6e5jR89s5pfLdxAyagh3HnJkcyeOIJTf7iAwqE5PPalE3RtyAGo3NPMzU8s56l3d3DI2HzuuPiIATEMek1FHV/47RI27GzgW2cdwhe6tKLTQUomGDM7DzjH3f/RzE6h7wTzL0STy3XB828COd0STKO7/7C346kFkzpa2yN86t6FbKpp5O/Xn8S23U386tUNPPXudtojzsxxw/j4zDF8/NDRzBo/fJ/P/S9aV803H32brbua+Nj0Il5aXZVWpzQS7ZnlO/iPvyyjsq6FS4+ZyDfPPISRPUxNlIraOyJU1rVQsaeZij0tlO+s52fz1zIkO4ufXnZUQkd1hiVVE8x3gc8C7UAuMAx4zN2v6KHsUuDL7r4weK5TZGlu/c4GzvnJy2RlGHUt7eTnZPHpYyby2eMmU1KYd8CvX9/Szu1PreTh1zYxtSiP5752sjqpY2hPcxs/eW4Nv1q4gbzsTP75jBl8Zu6kPvsVK/Y0M2LIoKScP622oZUHF27g1ws3sLup7UPbjp0ykp9edlTaXk+UkgnmQwfpowVjZjOAp4EpHgQTdPIvAY4Oir1JtJO/prdjKMGknr+8tZUHXt3Ap46ewCePLu5Xp/++enNTLQVDspkSg6QlH7Wmoo6bn1jOq2urOXjMUD41p5gzDxvL5FHR33dbR4Snl+/goUUbeX19DdlZGcwuHsExU6JDzI+bOiqh0/Ts2N3M/S+X8/Brm2hq6+CMmWM49ZDRjBmWw+j8XMYMy6VwaHZanRLrLq0SjJndApS5++PBtpuJ9rf8a7d9rgH+LXh6m7s/2NcxlGBEEsPdeXr5Dn46fy3Lt+0BoqPVSksKeHZFBRV7Wpg4cjCfnjORPc1tvL6+hmXb9tARcYryc7h6XgmfmTs5rsP1m9s6uGfBOu55cR0dEeeC2eP50snTmD5m4Axh75TyCSYelGBEEm9zTSNPL9/BM8srWLKplhMPKuTKEyZz8sGjPzTTc0NLO6+vr+HBhRt4aXUVedmZXD53EmfNGsfEgsEUDs0J7ZTmglWV3PT4cjZWN3L+keP55pkzmDiy/5PQphslmH5QghFJLpGI9ytJLN+2m/teKufJd7a/f2+j7KwMikcMZvSwHEbmZTNiSDYFQwZx8Jh8zjxs7D6fVmtq7WDhup38/o3NPLOigqlFedx6QXTo/ECnBNMPSjAiqW3H7mZWbt/DluAeRZtrG6mqa6G2sY3ahlZ2NbXREXFG5mVz6TET+cxxk5kwovc7e7a0d/Doki08t6KCheuqaWmPMDQniy+dMo3PfWxKUg42SIQwE0zse1VFRPbD2OG5jB3e+0itSMRZVF7Nrxdu4N4X13Hvi+s45/Bx3PR/DqMo/8M32qttaOULv1nC6xtqmDRyCJfPncTph4zhmCkFSixxpAQjIikhI8OYd1Ah8w4qZOuuJn67eCMPvLKeReuq+d7FR/DxmdHbFa/f2cDVD77Ott3N/OTS2Zx/5Pi0HgWWzHSKTERS1uqKOq5/5C1Wbt/D5XMnceZhY7n+kaVkmHH//5vDnMkDb2qgfaU+mH5QghEZmFraO/jRM6u57+Vy3GFaUR4PXnXsPt2efCBTH4yISC9ysjK54ZxDOXlGEc+tqOT606enxTT66UAJRkTSwgnTCjlhmoYdJ5OBM9+2iIjElRKMiIiEQglGRERCoQQjIiKhUIIREZFQKMGIiEgolGBERCQUSjAiIhKKtJkqxsyqgI3dVg8Hdu/jur0tFwI79zPMno69L2X6U5941WVvse6tzL7WpfvzzuWu6/Te9C/WvZXRe5PYz4C+yoVRlzx3L+pHTPvO3dP2Ady3r+v2tkz0ds8xi2dfyvSnPvGqy4HWZ1/r0kcduq7Te6P3Jqnfm/7UJZbvTdh/Z3t7pPspsif2Y11/lmMZz76U6U994lWX/r5Ob2X2tS7dnz/RS5n9pfem7/V6b+L3GdBXuWSqy16lzSmyeDGzMg9p5tF4S6e6QHrVJ53qAulVH9Wl/9K9BROG+xIdQAylU10gveqTTnWB9KqP6tJPasGIiEgo1IIREZFQKMGIiEgoBnSCMbMHzKzSzJbtx75zzOxdM1trZnebmXXZdp2ZrTKz5Wb2/dhG3Ws8Ma+Lmd1sZlvN7K3gcU7sI+81plDem2D7N8zMzSwud6cK6b251czeCd6XZ8xsfOwj7zGeMOryAzN7L6jPn81sROwj7zWmMOpzSfC/HzGz0AcDHEgdenm9K81sTfC4ssv6Pv+vehTmGOhkfwAnAUcDy/Zj39eB4wED/gacHaw/FXgOyAmej07hutwMfCNd3ptg20TgaaIX5Ramal2AYV3K/BNwbwrX5QwgK1j+HvC9VP47Aw4FZgALgNJkrUMQX0m3dSOB8uBnQbBc0Fd9+3oM6BaMu78E1HRdZ2bTzOzvZrbEzF42s0O672dm44j+gy/y6G/+IeDCYPOXgDvcvSU4RmW4tYgKqS4JE2J9fgx8C4jb6JYw6uLue7oUzSNO9QmpLs+4e3tQdDFQHG4tPhBSfVa6+6p4xB8cb7/q0IszgWfdvcbda4FngbP293NiQCeYXtwHXOfuc4BvAP/VQ5kJwJYuz7cE6wAOBj5mZq+Z2Ytmdkyo0fbtQOsC8JXg1MUDZlYQXqj9ckD1MbPzga3u/nbYgfbDAb83ZnabmW0GPgPcGGKsexOLv7NO1xD9dpxIsaxPovSnDj2ZAGzu8ryzXvtV36x+HnRAMLOhwAnAH7ucXszpqWgP6zq/QWYRbVoeBxwD/MHMpgZZP25iVJd7gFuD57cCPyT6ARB3B1ofMxsCfJvo6ZiEitF7g7t/G/i2md0AfAW4Kcah7lWs6hK81reBduB/YhnjvohlfRKlrzqY2dXA9cG6g4CnzKwVWO/uF9F7vfarvkowH5YB7HL32V1XmlkmsCR4+jjRD96uzfhiYFuwvAV4LEgor5tZhOiEclVhBt6DA66Lu1d02e9+4MkwA96LA63PNGAK8HbwT1cMvGlmx7r7jpBj7y4Wf2ddPQz8lQQkGGJUl6Az+Tzg9Hh/Gesm1u9NIvRYBwB3fxB4EMDMFgBXufuGLkW2AKd0eV5MtK9mC/tT37A7oJL9AZTQpXMMWAhcEiwbcGQv+71BtJXS2eF1TrD+i8AtwfLBRJublqJ1GdelzNeAR1L5velWZgNx6uQP6b2Z3qXMdcCjKVyXs4AVQFE8/77C/jsjTp38+1sHeu/kX0/0LExBsDyyP/XtMa5EvKHJ8gB+B2wH2ohm6H8g+i3378DbwR/9jb3sWwosA9YBP+ODWRGygd8G294ETkvhuvwGeBd4h+i3tnHxqEtY9elWZgPxG0UWxnvzp2D9O0QnLpyQwnVZS/SL2FvBIy4j4kKsz0XBa7UAFcDTyVgHekgwwfprgvdkLXD13urb10NTxYiISCg0ikxEREKhBCMiIqFQghERkVAowYiISCiUYEREJBRKMJLWzKw+zsf7pZnNjNFrdVh0tuRlZvbE3mYZNrMRZvaPsTi2SCxomLKkNTOrd/ehMXy9LP9gYsZQdY3dzH4NrHb32/ooXwI86e6z4hGfyN6oBSMDjpkVmdmfzOyN4DEvWH+smS00s6XBzxnB+qvM7I9m9gTwjJmdYmYLzOxRi97H5H86740RrC8NluuDCSnfNrPFZjYmWD8teP6Gmd3Sz1bWIj6YtHOomT1vZm9a9P4cFwRl7gCmBa2eHwRlvxkc5x0z+88Y/hpF9koJRgainwA/dvdjgIuBXwbr3wNOcvejiM5OfHuXfY4HrnT304LnRwFfBWYCU4F5PRwnD1js7kcCLwGf73L8nwTH3+t8TsE8WKcTnU0BoBm4yN2PJnr/oR8GCe5fgXXuPtvdv2lmZwDTgWOB2cAcMztpb8cTiRVNdikD0ceBmV1mmh1mZvnAcODXZjad6Eyxg7rs86y7d73nxuvuvgXAzN4iOhfUK92O08oHE4QuAT4RLB/PB/fSeBi4s5c4B3d57SVE780B0bmgbg+SRYRoy2ZMD/ufETyWBs+HEk04L/VyPJGYUoKRgSgDON7dm7quNLOfAi+4+0VBf8aCLpsbur1GS5flDnr+X2rzDzo5eyvTlyZ3n21mw4kmqi8DdxO9/0sRMMfd28xsA5Dbw/4GfNfdf7GPxxWJCZ0ik4HoGaL3TwHAzDqnNR8ObA2Wrwrx+IuJnpoDuHRvhd19N9HbIn/DzAYRjbMySC6nApODonVAfpddnwauCe4PgplNMLPRMaqDyF4pwUi6G2JmW7o8vk70w7o06PheQfQWCwDfB75rZq8CmSHG9FXg62b2OjAO2L23Hdx9KdGZcS8lekOuUjMrI9qaeS8oUw28Ggxr/oG7P0P0FNwiM3sXeJQPJyCRUGmYskicBXfXbHJ3N7NLgcvc/YK97SeSatQHIxJ/c4CfBSO/dpGg21CLhE0tGBERCYX6YEREJBRKMCIiEgolGBERCYUSjIiIhEIJRkREQvH/AT4s/8tfTarhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot(skip_end=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.406041</td>\n",
       "      <td>4.224589</td>\n",
       "      <td>0.284489</td>\n",
       "      <td>3:16:12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fine-tuning the last layers\n",
    "learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the model\n",
    "learn.save('fit_head')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load Model for later use (e.x. to improve accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('fit_head')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To complete the fine-tuning, we can then unfeeze and launch a new training.\n",
    "# This takes about 3-4 hours\n",
    "# 0.3 or more is usually pretty good --> Predict the next word in 1/3 of the time!\n",
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(10, 1e-3, moms=(0.8,0.7))\n",
    "learn.save('fine_tuned')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = \"I liked this movie because\"\n",
    "N_WORDS = 40\n",
    "N_SENTENCES = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I liked this movie because i was shocked by the positive reviews and received reviews from i - do - not i was sick of this movie . i 'm not especially my roommate who was really surprised . At the time i saw\n",
      "I liked this movie because it was not made when Xxbos The Xxbos Be an all - star movie by Van Damme and Van Damme 's career had me all over the window . \n",
      " \n",
      "  The\n"
     ]
    }
   ],
   "source": [
    "learn.predict(TEXT, N_WORDS, temperature=1.1, min_p = 0.001)\n",
    "print(\"\\n\".join(learn.predict(TEXT, N_WORDS, temperature=0.75) for _ in range(N_SENTENCES)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We have to save not only the model, but also its encoder, the part that's responsible for creating and updating the hidden state. \n",
    "#### For the next part, we don't care about the part that tries to guess the next word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifer Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, we'll create a new data object that only grabs the labelled data and keeps those labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas = (TextList.from_csv(path, vocab=data_lm.vocab) #grab all the text files in path\n",
    "             # Vocab variable: we want to make sure that it uses exactly the same vocab that are used for the language model. \n",
    "             # If word number 10 was the in the language model, we need to make sure that word number 10 is the in the classifier. \n",
    "             # Because otherwise, the pre-trained model is going to be totally meaningless\n",
    "             .split_by_folder(valid='test')\n",
    "             #split by train and valid folder (that only keeps 'train' and 'test' so no need to filter)\n",
    "             .label_from_folder(classes=['neg', 'pos'])\n",
    "             #label them all with their folders\n",
    "             .databunch(bs=bs))\n",
    "\n",
    "data_clas.save('data_clas.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas = load_data(path, 'data_clas.pkl', bs=bs)\n",
    "# or try this\n",
    "# data_clas = TextClasDataBunch.load(path, 'data_clas.pkl', bs=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj match 1 : xxmaj tag xxmaj team xxmaj table xxmaj match xxmaj bubba xxmaj ray and xxmaj spike xxmaj dudley vs xxmaj eddie xxmaj guerrero and xxmaj chris xxmaj benoit xxmaj bubba xxmaj ray and xxmaj spike xxmaj dudley started things off with a xxmaj tag xxmaj team xxmaj table xxmaj match against xxmaj eddie xxmaj guerrero and xxmaj chris xxmaj benoit . xxmaj according to the rules</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj warning : xxmaj does contain spoilers . \\n \\n  xxmaj open xxmaj your xxmaj eyes \\n \\n  xxmaj if you have not seen this film and plan on doing so , just stop reading here and take my word for it . xxmaj you have to see this film . i have seen it four times so far and i still have n't made up my</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos * * * xxup spoilers * * * * * * xxup spoilers * * * xxmaj continued ... \\n \\n  xxmaj from here on in the whole movie collapses in on itself . xxmaj first we meet a rogue program with the indication we 're gon na get ghosts and vampires and werewolves and the like . xxmaj we get a guy with a retarded accent talking</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj to be a xxmaj buster xxmaj keaton fan is to have your heart broken on a regular basis . xxmaj most of us first encounter xxmaj keaton in one of the brilliant feature films from his great period of independent production : ' xxmaj the xxmaj general ' , ' xxmaj the xxmaj navigator ' , ' xxmaj sherlock xxmaj jnr ' . xxmaj we recognise him as</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj anyone who visited drive - ins in the 1950s , 60s , and 70s , must have seen a film or two by xxmaj american xxmaj international xxmaj pictures , a distributor that resembled 1980s giant xxmaj cannon xxmaj films . xxmaj wherever movie - goers ventured , xxup aip would be right there to supply the latest en vogue titles - in the 50s came horror movies</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a text classifier learner\n",
    "learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5)\n",
    "# If you're overfitting, you can increase \"drop_mult\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (25000 items)\n",
       "x: TextList\n",
       "xxbos xxmaj story of a man who has unnatural feelings for a pig . xxmaj starts out with a opening scene that is a terrific example of absurd comedy . a formal orchestra audience is turned into an insane , violent mob by the crazy xxunk of it 's singers . xxmaj unfortunately it stays absurd the xxup whole time with no general narrative eventually making it just too off putting . xxmaj even those from the era should be turned off . xxmaj the cryptic dialogue would make xxmaj shakespeare seem easy to a third grader . xxmaj on a technical level it 's better than you might think with some good cinematography by future great xxmaj vilmos xxmaj zsigmond . xxmaj future stars xxmaj sally xxmaj kirkland and xxmaj frederic xxmaj forrest can be seen briefly .,xxbos xxmaj airport ' 77 starts as a brand new luxury 747 plane is loaded up with valuable paintings & such belonging to rich businessman xxmaj philip xxmaj stevens ( xxmaj james xxmaj stewart ) who is flying them & a bunch of xxup vip 's to his estate in preparation of it being opened to the public as a museum , also on board is xxmaj stevens daughter xxmaj julie ( xxmaj kathleen xxmaj quinlan ) & her son . xxmaj the luxury jetliner takes off as planned but mid - air the plane is hi - jacked by the co - pilot xxmaj chambers ( xxmaj robert xxmaj foxworth ) & his two accomplice 's xxmaj banker ( xxmaj monte xxmaj markham ) & xxmaj wilson ( xxmaj michael xxmaj pataki ) who knock the passengers & crew out with sleeping gas , they plan to steal the valuable cargo & land on a disused plane strip on an isolated island but while making his descent xxmaj chambers almost hits an oil rig in the xxmaj ocean & loses control of the plane sending it crashing into the sea where it sinks to the bottom right bang in the middle of the xxmaj bermuda xxmaj triangle . xxmaj with air in short supply , water leaking in & having flown over 200 miles off course the problems mount for the survivor 's as they await help with time fast running out ... \n",
       " \n",
       "  xxmaj also known under the slightly different tile xxmaj airport 1977 this second sequel to the smash - hit disaster thriller xxmaj airport ( 1970 ) was directed by xxmaj jerry xxmaj jameson & while once again like it 's predecessors i ca n't say xxmaj airport ' 77 is any sort of forgotten classic it is entertaining although not necessarily for the right reasons . xxmaj out of the three xxmaj airport films i have seen so far i actually liked this one the best , just . xxmaj it has my favourite plot of the three with a nice mid - air hi - jacking & then the crashing ( did n't he see the oil rig ? ) & sinking of the 747 ( maybe the makers were trying to cross the original xxmaj airport with another popular disaster flick of the period xxmaj the xxmaj poseidon xxmaj adventure ( 1972 ) ) & submerged is where it stays until the end with a stark dilemma facing those trapped inside , either suffocate when the air runs out or drown as the 747 floods or if any of the doors are opened & it 's a decent idea that could have made for a great little disaster flick but bad unsympathetic character 's , dull dialogue , lethargic set - pieces & a real lack of danger or suspense or tension means this is a missed opportunity . xxmaj while the rather sluggish plot keeps one entertained for 108 odd minutes not that much happens after the plane sinks & there 's not as much urgency as i thought there should have been . xxmaj even when the xxmaj navy become involved things do n't pick up that much with a few shots of huge ships & helicopters flying about but there 's just something lacking here . xxmaj george xxmaj kennedy as the jinxed airline worker xxmaj joe xxmaj patroni is back but only gets a couple of scenes & barely even says anything preferring to just look worried in the background . \n",
       " \n",
       "  xxmaj the home video & theatrical version of xxmaj airport ' 77 run 108 minutes while the xxup us xxup tv versions add an extra hour of footage including a new opening credits sequence , many more scenes with xxmaj george xxmaj kennedy as xxmaj patroni , flashbacks to flesh out character 's , longer rescue scenes & the discovery or another couple of dead bodies including the navigator . xxmaj while i would like to see this extra footage i am not sure i could sit through a near three hour cut of xxmaj airport ' 77 . xxmaj as expected the film has dated badly with horrible fashions & interior design choices , i will say no more other than the toy plane model effects are n't great either . xxmaj along with the other two xxmaj airport sequels this takes pride of place in the xxmaj razzie xxmaj award 's xxmaj hall of xxmaj shame although i can think of lots of worse films than this so i reckon that 's a little harsh . xxmaj the action scenes are a little dull unfortunately , the pace is slow & not much excitement or tension is generated which is a shame as i reckon this could have been a pretty good film if made properly . \n",
       " \n",
       "  xxmaj the production values are alright if nothing spectacular . xxmaj the acting is n't great , two time xxmaj oscar winner xxmaj jack xxmaj lemmon has said since it was a mistake to star in this , one time xxmaj oscar winner xxmaj james xxmaj stewart looks old & frail , also one time xxmaj oscar winner xxmaj lee xxmaj grant looks drunk while xxmaj sir xxmaj christopher xxmaj lee is given little to do & there are plenty of other familiar faces to look out for too . \n",
       " \n",
       "  xxmaj airport ' 77 is the most disaster orientated of the three xxmaj airport films so far & i liked the ideas behind it even if they were a bit silly , the production & bland direction does n't help though & a film about a sunken plane just should n't be this boring or lethargic . xxmaj followed by xxmaj the xxmaj concorde ... xxmaj airport ' 79 ( 1979 ) .,xxbos xxmaj this film lacked something i could n't put my finger on at first : charisma on the part of the leading actress . xxmaj this inevitably translated to lack of chemistry when she shared the screen with her leading man . xxmaj even the romantic scenes came across as being merely the actors at play . xxmaj it could very well have been the director who miscalculated what he needed from the actors . i just do n't know . \n",
       " \n",
       "  xxmaj but could it have been the screenplay ? xxmaj just exactly who was the chef in love with ? xxmaj he seemed more enamored of his culinary skills and restaurant , and ultimately of himself and his youthful exploits , than of anybody or anything else . xxmaj he never convinced me he was in love with the princess . \n",
       " \n",
       "  i was disappointed in this movie . xxmaj but , do n't forget it was nominated for an xxmaj oscar , so judge for yourself .,xxbos xxmaj sorry everyone , , , i know this is supposed to be an \" art \" film , , but wow , they should have handed out guns at the screening so people could blow their brains out and not watch . xxmaj although the scene design and photographic direction was excellent , this story is too painful to watch . xxmaj the absence of a sound track was brutal . xxmaj the l xxrep 4 o xxrep 5 n g shots were too long . xxmaj how long can you watch two people just sitting there and talking ? xxmaj especially when the dialogue is two people complaining . i really had a hard time just getting through this film . xxmaj the performances were excellent , but how much of that dark , sombre , uninspired , stuff can you take ? xxmaj the only thing i liked was xxmaj maureen xxmaj stapleton and her red dress and dancing scene . xxmaj otherwise this was a ripoff of xxmaj bergman . xxmaj and i 'm no fan f his either . i think anyone who says they enjoyed 1 1 / 2 hours of this is , , well , lying .,xxbos xxmaj when i was little my parents took me along to the theater to see xxmaj interiors . xxmaj it was one of many movies i watched with my parents , but this was the only one we walked out of . xxmaj since then i had never seen xxmaj interiors until just recently , and i could have lived out the rest of my life without it . xxmaj what a pretentious , ponderous , and painfully boring piece of 70 's wine and cheese tripe . xxmaj woody xxmaj allen is one of my favorite directors but xxmaj interiors is by far the worst piece of crap of his career . xxmaj in the unmistakable style of xxmaj ingmar xxmaj berman , xxmaj allen gives us a dark , angular , muted , insight in to the lives of a family wrought by the psychological damage caused by divorce , estrangement , career , love , non - love , xxunk , whatever . xxmaj the film , intentionally , has no comic relief , no music , and is drenched in shadowy pathos . xxmaj this film style can be best defined as expressionist in nature , using an improvisational method of dialogue to illicit a \" more pronounced depth of meaning and truth \" . xxmaj but xxmaj woody xxmaj allen is no xxmaj ingmar xxmaj bergman . xxmaj the film is painfully slow and dull . xxmaj but beyond that , i simply had no connection with or sympathy for any of the characters . xxmaj instead i felt only contempt for this parade of shuffling , whining , nicotine stained , martyrs in a perpetual quest for identity . xxmaj amid a backdrop of cosmopolitan affluence and baked xxmaj brie intelligentsia the story looms like a fart in the room . xxmaj everyone speaks in affected platitudes and elevated language between cigarettes . xxmaj everyone is \" lost \" and \" struggling \" , desperate to find direction or understanding or whatever and it just goes on and on to the point where you just want to slap all of them . xxmaj it 's never about resolution , it 's only about interminable introspective babble . xxmaj it is nothing more than a psychological drama taken to an extreme beyond the audience 's ability to connect . xxmaj woody xxmaj allen chose to make characters so immersed in themselves we feel left out . xxmaj and for that reason i found this movie painfully self indulgent and spiritually draining . i see what he was going for but his insistence on promoting his message through xxmaj prozac prose and distorted film techniques jettisons it past the point of relevance . i highly recommend this one if you 're feeling a little too happy and need something to remind you of death . xxmaj otherwise , let 's just pretend this film never happened .\n",
       "y: CategoryList\n",
       "neg,neg,neg,neg,neg\n",
       "Path: C:\\Users\\bokhy\\.fastai\\data\\imdb;\n",
       "\n",
       "Valid: LabelList (25000 items)\n",
       "x: TextList\n",
       "xxbos xxmaj once again xxmaj mr. xxmaj costner has dragged out a movie for far longer than necessary . xxmaj aside from the terrific sea rescue sequences , of which there are very few i just did not care about any of the characters . xxmaj most of us have ghosts in the closet , and xxmaj costner 's character are realized early on , and then forgotten until much later , by which time i did not care . xxmaj the character we should really care about is a very cocky , overconfident xxmaj ashton xxmaj kutcher . xxmaj the problem is he comes off as kid who thinks he 's better than anyone else around him and shows no signs of a cluttered closet . xxmaj his only obstacle appears to be winning over xxmaj costner . xxmaj finally when we are well past the half way point of this stinker , xxmaj costner tells us all about xxmaj kutcher 's ghosts . xxmaj we are told why xxmaj kutcher is driven to be the best with no prior inkling or foreshadowing . xxmaj no magic here , it was all i could do to keep from turning it off an hour in .,xxbos xxmaj this is an example of why the majority of action films are the same . xxmaj generic and boring , there 's really nothing worth watching here . a complete waste of the then barely - tapped talents of xxmaj ice - t and xxmaj ice xxmaj cube , who 've each proven many times over that they are capable of acting , and acting well . xxmaj do n't bother with this one , go see xxmaj new xxmaj jack xxmaj city , xxmaj ricochet or watch xxmaj new xxmaj york xxmaj undercover for xxmaj ice - t , or xxmaj boyz n the xxmaj hood , xxmaj higher xxmaj learning or xxmaj friday for xxmaj ice xxmaj cube and see the real deal . xxmaj ice - t 's horribly cliched dialogue alone makes this film grate at the teeth , and i 'm still wondering what the heck xxmaj bill xxmaj paxton was doing in this film ? xxmaj and why the heck does he always play the exact same character ? xxmaj from xxmaj aliens onward , every film i 've seen with xxmaj bill xxmaj paxton has him playing the exact same irritating character , and at least in xxmaj aliens his character died , which made it somewhat gratifying ... \n",
       " \n",
       "  xxmaj overall , this is second - rate action trash . xxmaj there are countless better films to see , and if you really want to see this one , watch xxmaj judgement xxmaj night , which is practically a carbon copy but has better acting and a better script . xxmaj the only thing that made this at all worth watching was a decent hand on the camera - the cinematography was almost refreshing , which comes close to making up for the horrible film itself - but not quite . 4 / 10 .,xxbos xxmaj first of all i hate those moronic rappers , who could'nt act if they had a gun pressed against their foreheads . xxmaj all they do is curse and shoot each other and acting like xxunk version of gangsters . \n",
       " \n",
       "  xxmaj the movie does n't take more than five minutes to explain what is going on before we 're already at the warehouse xxmaj there is not a single sympathetic character in this movie , except for the homeless guy , who is also the only one with half a brain . \n",
       " \n",
       "  xxmaj bill xxmaj paxton and xxmaj william xxmaj sadler are both hill billies and xxmaj xxunk character is just as much a villain as the gangsters . i did'nt like him right from the start . \n",
       " \n",
       "  xxmaj the movie is filled with pointless violence and xxmaj walter xxmaj hills specialty : people falling through windows with glass flying everywhere . xxmaj there is pretty much no plot and it is a big problem when you root for no - one . xxmaj everybody dies , except from xxmaj paxton and the homeless guy and everybody get what they deserve . \n",
       " \n",
       "  xxmaj the only two black people that can act is the homeless guy and the junkie but they 're actors by profession , not annoying ugly brain dead rappers . \n",
       " \n",
       "  xxmaj stay away from this crap and watch 48 hours 1 and 2 instead . xxmaj at lest they have characters you care about , a sense of humor and nothing but real actors in the cast .,xxbos xxmaj not even the xxmaj beatles could write songs everyone liked , and although xxmaj walter xxmaj hill is no mop - top he 's second to none when it comes to thought provoking action movies . xxmaj the nineties came and social platforms were changing in music and film , the emergence of the xxmaj rapper turned movie star was in full swing , the acting took a back seat to each man 's overpowering regional accent and transparent acting . xxmaj this was one of the many ice - t movies i saw as a kid and loved , only to watch them later and cringe . xxmaj bill xxmaj paxton and xxmaj william xxmaj sadler are firemen with basic lives until a burning building tenant about to go up in flames hands over a map with gold implications . i hand it to xxmaj walter for quickly and neatly setting up the main characters and location . xxmaj but i fault everyone involved for turning out xxmaj lame - o performances . xxmaj ice - t and cube must have been red hot at this time , and while i 've enjoyed both their careers as rappers , in my opinion they fell flat in this movie . xxmaj it 's about ninety minutes of one guy ridiculously turning his back on the other guy to the point you find yourself locked in multiple states of disbelief . xxmaj now this is a movie , its not a documentary so i wo nt waste my time recounting all the stupid plot twists in this movie , but there were many , and they led nowhere . i got the feeling watching this that everyone on set was xxunk of confused and just playing things off the cuff . xxmaj there are two things i still enjoy about it , one involves a scene with a needle and the other is xxmaj sadler 's huge 45 pistol . xxmaj bottom line this movie is like domino 's pizza . xxmaj yeah ill eat it if i 'm hungry and i do n't feel like cooking , xxmaj but i 'm well aware it tastes like crap . 3 stars , meh .,xxbos xxmaj brass pictures ( movies is not a fitting word for them ) really are somewhat brassy . xxmaj their alluring visual qualities are reminiscent of expensive high class xxup tv commercials . xxmaj but unfortunately xxmaj brass pictures are feature films with the pretense of wanting to entertain viewers for over two hours ! xxmaj in this they fail miserably , their undeniable , but rather soft and flabby than steamy , erotic qualities non withstanding . \n",
       " \n",
       "  xxmaj senso ' 45 is a remake of a film by xxmaj luchino xxmaj visconti with the same title and xxmaj alida xxmaj valli and xxmaj farley xxmaj granger in the lead . xxmaj the original tells a story of senseless love and lust in and around xxmaj venice during the xxmaj italian wars of independence . xxmaj brass moved the action from the 19th into the 20th century , 1945 to be exact , so there are xxmaj mussolini murals , men in black shirts , xxmaj german uniforms or the tattered garb of the partisans . xxmaj but it is just window dressing , the historic context is completely negligible . \n",
       " \n",
       "  xxmaj anna xxmaj xxunk plays the attractive aristocratic woman who falls for the amoral xxup ss guy who always puts on too much lipstick . xxmaj she is an attractive , versatile , well trained xxmaj italian actress and clearly above the material . xxmaj her wide range of facial expressions ( signalling boredom , loathing , delight , fear , hate ... and ecstasy ) are the best reason to watch this picture and worth two stars . xxmaj she endures this basically trashy stuff with an astonishing amount of dignity . i wish some really good parts come along for her . xxmaj she really deserves it .\n",
       "y: CategoryList\n",
       "neg,neg,neg,neg,neg\n",
       "Path: C:\\Users\\bokhy\\.fastai\\data\\imdb;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(60000, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(60000, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x000001482542F4C8>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=WindowsPath('C:/Users/bokhy/.fastai/data/imdb'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True, silent=False)], callbacks=[RNNTrainer\n",
       "learn: ...\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): Embedding(60000, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(60000, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 1152, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1152, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.2, inplace=False)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace=True)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1, inplace=False)\n",
       "      (6): Linear(in_features=50, out_features=2, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True, silent=False)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.load_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='93' class='' max='1250', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      7.44% [93/1250 01:06<13:52 1.4555]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU1dnA8d8zWcmekAVIIAkQlgSQHdwAUQGXatUuYBftZt+2amtrW237Wqtv98W21i62amtbpS5YN1RQRJFC2beELQkBEiArWSZ7Zs77x8yEIZksQCaz5Pl+PvNh7r3nzjyXTPLMWe45YoxBKaWU6sri6wCUUkr5J00QSimlPNIEoZRSyiNNEEoppTzSBKGUUsqjUF8HMFCSk5NNVlaWr8NQSqmAsn379ipjTIqnY0GTILKysti2bZuvw1BKqYAiIkd7OqZNTEoppTzSBKGUUsojTRBKKaU80gShlFLKI00QSimlPNIEoZRSyiNNEEoppTzSBKGUUgFs1Y5Snt92HG8s3aAJQimlApTdbvjlmkP8e1cZIjLgr68JQimlAtSGwirKaptZMXeMV15fE4RSSgWolVuOkRQdztW5aV55fU0QSikVgCoaWlhbUM5HZmUQERrilffQBKGUUgHohe2ldNgNy+eM9tp7aIJQSqkAY7cbVm45zrzsJMamxHjtfTRBKKVUgNlUXM2xmiZuneedzmkXTRBKKRVgntlyjISoMJbmjfDq+2iCUEqpAFJlbWVN/ilunpFBZJh3OqddNEEopVQAWbWjlHabYcVc73VOu2iCUEqpALK5uIac1Bhy0mK9/l6aIFS/vbyrjMKKBl+HodSQVlxpJSfNeyOX3GmCUP1yoraZr/1rF79/t8jXoSg1ZLV22DhW08Q4Lw5tdacJQvXLSzvLMAb2najzdShKDVnHqpuwGzRBKP9hjOHF7aUAFFZYaW6z+TgipYamokorAGNTogfl/TRBqD7tPF5LcVUjV+emYTdQcLLe1yEpNSQVVTYCePXuaXeaIFSfXtxeSmSYhW8tnQhAvjYzKeUTRZVW0uIiiIkIHZT30wShetXSbuPV3SdYljeC8akxDI8OZ2/puSeIprYOVu0oxW4f+FWvlBoqiisbB63/AbycIERkmYgcFJFCEbnPw/FHRGSX83FIRGrdjv1MRPJFZL+I/Fa8sVyS6tM7+yuob+ngllkZiAh56fHsO3HuTUyPrivk68/tZkNhlReiVCr4GWMoqrQGR4IQkRDgMeAaIBdYISK57mWMMfcYY6YbY6YDjwKrnOdeAlwKTAOmAHOAhd6KVfXsxR2ljIiL5JJxyQBMTY/jcHkDLe3976iubWrj6f+UAPDmvpPeCFOpoFdlbaOhpWPQOqjBuzWIuUChMabYGNMGrARu7KX8CuBZ53MDRALhQAQQBpR7MVblQWVDK+8dquSmmemEWBwVuCmj4umwGw6e6v8Nc09uLKGxzca0jHjW5Jdj02Ympc6ZawRTUNQggHTguNt2qXNfNyKSCWQD6wCMMZuAd4GTzsdbxpj9Hs67Q0S2ici2ysrKAQ5fvbyrDJvdcMvMjM59U9Ljgf7fD1Hf0s5TG4+wNC+N/1k4jurGNrYcqfFKvEoFs+LOEUzBUYPw1GfQ01fH5cALxhgbgIiMByYDGTiSymIRWdDtxYx53Bgz2xgzOyUlZYDCVi7/3lXGRaMTGJ965htLRuIw4oeFsa+sfwni6f+U0NDSwV2Lc1g0MYXIMIs2Myl1HooqrUSGWRgVP2zQ3tObCaIUcJ9uMAM40UPZ5ZxpXgK4CdhsjLEaY6zAG8B8r0SpPKpsaGVfWT1L885eDF1EmJoez76yvjuqG1s7eOKDIyyelMqU9HiiwkNZOCGFN/ad0tFMSp2jokorY5NjsFgGb7yONxPEViBHRLJFJBxHEnilayERmQgkApvcdh8DFopIqIiE4eig7tbEpLxno3O00WXjk7sdy0uP4+CpBto67L2+xj82H+V0Uzt3LR7fue/aqSOpaGhl5/HTAxuwUkGuuLJxUJuXwIsJwhjTAdwJvIXjj/tzxph8EXlIRG5wK7oCWGmMcf9K+QJQBOwFdgO7jTGveitW1d2Gw1UkRIWRNyq+27Epo+Jps9k5VN5zR3Vzm40/byjm8pxkZoxJ7Ny/eFIq4SEW3th7yitxKxWMWtptHD89eJP0uXj1djxjzGpgdZd9D3TZftDDeTbgi96MTfXMGMMHhZVcOi65c/SSu6mujuqyus5O666e+KCYKmsbd1+Zc9b+2MgwLstJ5o19p/judZPR21uU6tvR6iaMGdwOavByglCBqbDCSnl9K5fldG9eAhiTFEVsRGiPI5lO1bXw+/VFLMsbwZyspG7Hl00ZwboDFewrq2dqRjw2u2MywLX7y2lus9HcbqOpzca09Hh+cstUj0nkj+8VkRYXwU0zMrodUyrY+GKIK2iCUB5sONxz/wOAxSLkpcext4eO6p+9eYAOm+E71072eHxJbhrfsQir952koaWdh1/fz/6T9WQOj2J4dDjDwkOwG8O/th3nhumjuLRLHPtP1vOTNw4QGxnK4klpxA8Lu4CrVcr/FQ/yLK4umiBUNx8UVpE1PIrRSVE9lpkyKp6nNx+l3WYnLORMV9bOY6dZtbOMLy0ax5jhns9PiArn4nHDeWLDEf6wvoj0hGE8umIG108b2VlbaGm3ccUv1vPLNQe5ZNzws2oRv1p7iKjwEBpaOnhq4xG+dtWEAbpypfxTUWUjo+IjiQof3D/ZOlmfOktbh53NxdU9Ni+5TM2Ip63DTmGFtXOfMYaHXisgJTaCr1wxvpezYcXcMURHhPDNpRN55xsL+dBFo85KApFhIdy5eDw7jtWy/tCZmyB3Ha9lbUE5X1o4jqtz03jygyPUt7Sf59UqFRiKK62MSx3c5iXQBKG62HnsNE1tNi4b3/uNh67RTb95+zDrDpTT1NbBy7tOsPNYLd9cOrHP6YivnTqSnQ8s4StXjCcyLMRjmY/OGk1G4jB+teYQrkFuv1xzkKTocD5zWTZfvTKH+pYO/rax5NwvVKkA4Zikr5GxyYPbvATaxKS6+KCwCovAxeOG91pubHI0H7poFGsLTvFm/inCQyyEhjhuovvIzIHpOA4PtXD3lTl864U9rCkoJ35YGBsOV/G96yYTExHKlPR4rpyUyl8+OMLtl2YRG6l9ESr4VDa0Ym3t8EkNQhOEOsuGw1VcNDqhz45fi0V4dMUMWtptbCs5zfuHK9l57DTfuy53QO/0vHlGOn9YX8Qjaw8RExFKWlwEn5yf2Xn8q1flcMPvNvL0pqN9NmspFYgKfTSCCTRBKDd1Te3sKa3lznP4QxsZFsJlOcl99lmcr9AQC1+9Moev/WsXAA9/eMpZTVLTMhK4YmIKf95QzG2XZA3aSltKDZYiH0zS56J9EEHs3QMVLP7FerYf7d/sqZuKq7AbuCzHvyY+/NBFo5iQFsPopGF8fPbobsfvvjKH2qZ2fvBK/jmtU6FUICiutBIVHsKIuMhBf29NEEHsvUOVFFc1cuuf/8sbe/ueQXXD4Sqiw0OYMSZhEKLrvxCL8I/Pz+P5L15CeGj3j+yMMYncsWAsz28v5brfbmDnMZ3nSQWPIuccTL6YdUATRBA7VtNE1vAo8kbF8eVndvDEB0d6Lb+pqJp5Y4efdV+Dv0iNjWREfM/foL5z7WT+/rm5NLfZuOUP/+Fnbx6gtUNrEyrwFTtncfUF//tLoAZMSXUjk0bE8cwX5rMkN42HXyvgJ28c8Fi2vL6F4qpGLh7b++glf3Z5Tgpv3rOAj8zK4Pfri1j6yPu8XVDO2fNAQofN3nlnqlL+rKXdRllts086qEETRNCy2Q2lNc1kDo8iMiyE339iFjfPSOdP7xdRbW3tVn5zcTUA8wM4QQDERYbxs49cxN8/N5fQEAuff3obtz21lYOnGvjgcBX3r9rLvB+9w+Jfvsfre3ThIuXfjlQ1+mSSPhdNEEHqZF0zbTY7mcMdH6wQi3D7pVkYA+8f7r486+biGmIjQskdFTfYoXrF5TkpvPHVy/nf63PZefQ0S3/9Pp984r+8vKuMi8cNZ3TSMP70flG32oVS/sS1zKivahA6JjBIHatuAiDTbT6kKaPiSY4JZ/3Bym6zoP63uJq52Ukep/cOVGEhFj53WTY3Th/Fc9uOMzY5xrnsaQj//O9RvvvSPrYcqWFegNeaVPByzeKa7YO7qEFrEEHraE33BGGxCAtyUnjvUCU2tyU/Xf0Pgd681JPkmAi+vGg8y6aM6LyH4paZGSRFh/PnDb133CvlS8WVVtIThjEs3PN0NN6mCSJIlVQ3EhYijOyywPmiSanUNrWzu7S2c1+w9D+ci8iwED45P5N3DpRrh7XyW0U+WGbUnSaIIHWsuonRSVHdmowW5CRjEVh/8Ew/xObiamIjg6f/ob8+fXEmYSGWPof/KuULxhjHLK4+6n8ATRBBq6S6iUwP6zkkRIUzY0wi6w9WdO7bXFzDvCDrf+iP5JgIbp6RzgvbSz2O7FLKl8rrW2lsszFOaxBqIBljOFbd2DmCqatFE1LYU1pHlbWVU3UtHAni/oe+fP7ybFo77Pxj8zFfh6LUWc6sIue7GoSOYgpCVdY2GttsZ3VQu1s0MZVfrj3E+4cqO2sNQzVBjE+N5YqJKTy9qYSo8BCGx4STHBNB1vDoHlfEU2owFFX5dograIIISsdqHB+srB5qEHmj4kiOieDdg5XERIQQGxnK5JFDq//B3VevmsBtT27hh6v3d+4LtQjvf+sKRiUM6+VMpbynqMJKdHgIaXERPotBE0QQKqlyDHHt6RuwxSIsnJDC2/sdi/AMxf4Hd9NHJ7DrgatpaO2g2trGofIGvvj37byVf4rPXJrd67nNbTbKaptIio4gKTp8kCJWQ0FxVSPZPpqkz0UTRBA6WtOERSAjsedvv4smpvDijlLqmtv59MWZPZYbKkSEuMgw4iLDyE6OZkJaDG/u85wg3j9UySNvH+J4TTNVzs7trOFRrLlnocfZZpU6H0UVVmZnJfo0Bv00B6Fj1Y2MjB9GRGjPN9csyEnBVWkYqv0PvVmWN4KtJTXdRjfZ7YYHXt7HqboWrpyUyr1LJvDVK3MoqW7iX9uO+yhaFWya22ycqGv22SyuLpogglBJdRNZyb13sMZHhTFzTOKQ73/oydIpI7AbeHt/+Vn7NxVXU1LdxLeWTeSnH5nGnYtz+NpVOczJSuS37xymqa3DRxGrYOKapG9cqu+GuIImiKB0rKaJMUl9f7AevCGP366YMaT7H3qSOzKO0UnDeHPfqbP2P7PlGAlRYVwzZWTnPhHh28smUdnQyl//UzLIkapgVFzlHOKqNQg1kOpb2qlpbOtxiKu7KenxXDExdRCiCjwiwrK8EWwsrKa+pR2AKmsra/JPccvMjLPWxQaYnZXEVZNT+eP6Iuqa2n0RsgoiRRWNiPhukj4XryYIEVkmIgdFpFBE7vNw/BER2eV8HBKRWrdjY0RkjYjsF5ECEcnyZqzBwjWLa5aO4b9gy6aMoM1m590DjrvOn99WSrvNsGLuGI/l7106kYbWDv7wXtFghqmCUHGVlVHxvpukz8VrCUJEQoDHgGuAXGCFiOS6lzHG3GOMmW6MmQ48CqxyO/w08HNjzGRgLlCB6lNJteMeiP40ManezRidSEpsBG/ln8JuNzy75Rhzs5MYn+q52j9pRBw3TU/nqY1HOFXXMsjRqmBS7ONJ+ly8WYOYCxQaY4qNMW3ASuDGXsqvAJ4FcCaSUGPMWgBjjNUY0+TFWIPGUQ/rQKjzY7EIS/PSePdAJe8cqOBYTROfmOe59uByz9UTsBvD7949PEhRqmDQ1mHvfO4Pk/S5eDNBpAPu4/5Knfu6EZFMIBtY59w1AagVkVUislNEfu6skXQ97w4R2SYi2yoru6+SNhQdq24iOSaC6Ai9xWUgLMsbSXO7je+8tJfEqDCWTRnRa/nRSVF8eHo6q3aUYW3VEU2qb1tLapj4v29w5zM7KKyw+sUkfS7eTBCehsb0tL7jcuAFY4zNuR0KXA7cC8wBxgK3d3sxYx43xsw2xsxOSUm58IiDQEl1o/Y/DKB5Y5OIHxZGZUMrH5mV0eu9JS7L546hqc3Gq7tPDEKEKtAVnKjHGFhbUM6SR97j7pU7Ad/OweTizQRRCox2284AevqNWY6zecnt3J3O5qkO4N/ATK9EGWSO1TTpJHMDKCzEwlWT0wB67JzuauaYBCakxbByi84Qq/pWXt9CqEX44NuL+dxl2ew+XosIPfZ1DSZvtkNsBXJEJBsow5EEbu1aSEQmAonApi7nJopIijGmElgMbPNirEGhpd3GybqWHifpU+fn3qUTWJKX1u9pl0WE5XPG8NBrBRScqB9yCzGpc1Ne30pqbAQpsRF897pcPn/5WEqqGkmNi/R1aN6rQTi/+d8JvAXsB54zxuSLyEMicoNb0RXASmOMcTvXhqN56R0R2YujuerP3oo1WBz3sA61unAj44exNK/3voeubp6ZTniohZVbtRahelfR0HJWMkiLi2Sen0x/49WeTGPMamB1l30PdNl+sIdz1wLTvBZckDHGsKbAMS1ETwsFqcGTEBXONVNG8NLOMu6/ZrLPx7Mr/1Ve3+LzG+J6ondSB4HjNU18+skt/Pytg8wfm0Suzq3kF5bPGUNDSwer9570dSjKj5XXt5LmB81JnuhYyABmjOHJjSX84q2DWAQevjGPT8zLxKJzK/mF+WOTyE6OZuXWY9wyK8PX4Sg/1NJuo6653W8ThNYgAti7Byt4+LUC5o9NYu3XF/Kpi7M0OfgREeHjc0azteQ0hRUNvg5H+aGKesd08qmxvls1rjeaIALY2oJyYiJC+dOnZuvSmH7qlpkZhFqE57eV+joU5YfKGxxTsmgNQg0oYwzrDlRweU6yrmLmx1JiI7h0fDJv5p/CbaCeUoCjgxo0QagBln+invL6VhZP0um6/d3VuWkcrW7icIXV16EoP1PubGJKi9MmJjWA1h2oQAQW6XoOfs91J/bagvI+SqqhpqK+hfBQC/HDwnwdikeaIALUOwcquCgjgRQ/7dxSZ4yIj2RaRrwmCNVNeX0LaXERiPjn4BJNEAGosqGVPaW1XKnNSwHj6slp7DpeS0W9rhOhziivbyUt1j/7H0ATREBaf7ACY+AKTRAB4+o8RzPT2/t13St1RnlDi992UIMmiIC07kAFaXER5OkkcAFjYloso5OG8fZ+bWZSZ1TUt5Lqpx3UoAki4LR12NlwuIrFk9L8tt1SdSciXDU5jQ8Kq2jUhYQUYG3twNraoTUINXC2ltRgbe3Q/ocAdHVumjPB6+qHis7+KH8d4gqaIALOO/srCA+1cMl4/5gOWPXf3CzH6nRrdDSTAk65EoQfd1LrZH2DbE9pLbuP15I5PJrs5GhGJQwjpJ/zJxljeOdAOZeMG05UuP7oAk1oiIXFk1JZd6CCDpud0BD9fjaUdc7D5MdNTPpXZhAdKm/gY3/aREu7vXNfeIiFGWMSuGH6KK6ZMpKk6PAezy+qbORodROfuyx7MMJVXnDV5DRe2lnG9qOn/WZRGOUb5QHQxKQJYpC0tNu485kdRIeHsupL86hvaaekqpGiSivrDlTw3Zf28cDL+Vw2Ppn/vT7X43q0b+WfAhxt2SowLZyYQniIhTUF5Zoghrjy+laiwkOIifDfP8P+G1mQ+cGrBRwqt/K3z87tXKN4vvMPxHeuncz+kw28svsE/9x8lB+v3s8Tt8/p9hpr8k9x0egERsbrzK2BKiYilEvHD2dNwSm+d91kHYk2hLnugfDnz4A2gg6C1/ac4Nktx/ifheNYOCGl23ERIXdUHPddM4lb549h/aFKqqytZ5Upq21md2kdy85xbWTlf5bkjeB4TTP7T+oaEUNZhXOaDX+mCWKA2e2G4zVNHKlq5HB5A5uLq7n/xb3MHJPAN5ZM6PP8m2dkYLMbXt194qz9a5zNS0vztHkp0F01OQ0RWFNwytehKB/y56VGXbSJaYA9/HoBT20sOWtfXGQov10xg7B+jFqZOCKWvFFxrNpRxmcuPdMZ/Vb+KSakxTA2pXvfhAosKbERzBqTyFv55Xztqr6/NKjgY4xxTtSnCWLIqGtuZ+WW4yyamMKHp6cTGiKEWoQp6fFkJEb1+3VumpHO/72+n8KKBsanxlJtbWXLkRruvGK8F6NXg2lp3gh+uHo/x2uaGJ3U/8+GCg71zR20dtj9dqlRF21iGkDPbztOc7uNe5dM5MMz0rl+2iiWTRl5TskB4IbpowixCKt2lAHw9v5y7MbRdq2CwxJnU6FrZJoaWvx9qVEXTRADxGY3/G1TCXOyEpmSHn9Br5UaG8nlOcn8e2cZdrvhzX2nyEgcppPzBZHM4dFMGhGrd1UPUf6+1KiLJogBsu5ABcdrmrn9koG5ie3mmRmcqGvhnQMVbCysZlneCL8eDqfO3ZK8EWwrqaG6y4g1Ffz8falRF00QA+Sv/znCyPjIzqaDC7UkN42YiFC+9++9tNnsLJuizUvBZkluGnaDTgE+BLlqEKl+PA8TaIIYEIfKG9hYWM0n52f2a6RSf0SGhXDNlBGU17eSHBPBzDGJA/K6yn/kjYojPWEYa/I1QQw1FfUtxEWGMiw8xNeh9Kpff81EZJyIRDifLxKRu0UkwbuhBY6//qeE8FALK+aOGdDXvXlmBuDo0LT0c0I/FThEhKV5I9hQWIVV14gYUgLhHgjofw3iRcAmIuOBJ4Bs4Jm+ThKRZSJyUEQKReQ+D8cfEZFdzschEantcjxORMpE5Hf9jHPQ1TW1s2pHKR+ePqrXifbOx7zsJL5x9QS+uGDsgL6u8h9L8hxrRHz8T5u4f9Ve/rrxCNuPnvZ1WMrL/H2pUZf+3gdhN8Z0iMhNwK+NMY+KyM7eThCREOAx4GqgFNgqIq8YYwpcZYwx97iVvwuY0eVlHgbe62eMPvHy7jJa2u3cdknWgL+2xSLcdWXOgL+u8h9zspL48qJxbD96mjf2neTZLe0A/OETM7lm6kgfR6e8paK+lXljo30dRp/6myDaRWQFcBvwIee+sD7OmQsUGmOKAURkJXAjUNBD+RXA910bIjILSAPeBGb3M85Bl19WT3JMOHmjLmxoqxqaQizCt5ZNAhx311ZaW7npsf/w7NbjmiCClN1uqAiQGkR/m5g+A1wM/NAYc0REsoF/9HFOOnDcbbvUua8bEcnE0Wy1zrltAX4JfLOf8flMUaVVp79QA0JESI2N5OaZ6XxwuLJzpIsKfB02O8YYAE43tdFuM6T5+V3U0M8ahLNZ6G4AEUkEYo0xP+njNE+9qqaHssuBF4wxNuf2l4HVxpjjvY39F5E7gDsAxowZ2A7i/jDGUFhp5Zop+k1PDZybZ2bw6LpCXtpZxv8sHOfrcNQFqmtq57KfrqO53UZidDjRzpFLgVCD6FeCEJH1wA3O8ruAShF5zxjz9V5OKwVGu21nACd6KLsc+Irb9sXA5SLyZSAGCBcRqzHmrI5uY8zjwOMAs2fP7in5eE1NYxu1Te2MS/H/tkQVOLKTo5k5JoEXt5fyxQVj9QbJAHeyvpmG1g6W5qWRFB1OTWMbY1NimJ2V5OvQ+tTfPoh4Y0y9iHweeMoY830R2dPHOVuBHGdzVBmOJHBr10IiMhFIBDa59hljPuF2/HZgdtfk4A+KKhsBPK7+ptSFuGVWBt99aR/7yuqZmqH9W4Gs0TmE+dZ5mR7Xg/Fn/e2DCBWRkcDHgNf6c4IxpgO4E3gL2A88Z4zJF5GHROQGt6IrgJXG1UAXQIoqrQCM0z4INcCunzaK8FALL+4o9XUo6gJZWx0t5zER/n1TnCf9rUE8hOMP/UZjzFYRGQsc7uskY8xqYHWXfQ902X6wj9f4K/DXfsY5qIoqrESEWkhP0CVA1cCKHxbG1blpvLyrjO9cO5nwUJ30IFC5ahDRfrz2dE/69akzxjxvjJlmjPmSc7vYGHOLd0Pzf4XOEUx6l7Pyho/MzOB0UzvvHqzwdSjqArjuko8OD9IEISIZIvKSiFSISLmIvCgiGd4Ozt8VVVq1/0F5zeU5ySTHRPDidm1mCmSuGkRMsNYggKeAV4BROO5leNW5LygUVjTQ1mE/p3Na2m2Unm7WEUzKa0JDLHx4+ijePVjBidpmX4ejzlPQNzEBKcaYp4wxHc7HX4HA6o7vQVGllWW/3sATHxw5p/OOVDVijHZQK+/6xPxMwkMs3PbkFk43tvk6HHUerK02wkMsAdmP1N+Iq0TkkyIS4nx8Eqj2ZmCDZVxKDIsnpfLbdw5Terqp3+cVVugIJuV92cnR/OW2ORytaeL2p7borK8BqLG1g+gAHMEE/U8Qn8UxxPUUcBL4CI7pN4LC92/IA+AHr/Y0TVR3RZVWRGCsNjEpL7t43HB+f+tM9p2o5wt/20ZLu63vk5TfcCSIwGtegv6PYjpmjLnBGJNijEk1xnwYuNnLsQ2a9IRhfPWqHNYWlPN2lzWC7XbDkarGbucUVTaSkTiMyLDA/GagAstVuWn84qPT2FRczV3P7sRuD7jbhoYsa2tHQHZQw4WtKNfbNBsB53OXZZOTGsP3X8mnuc3xDa2wwsrH/rSJK36xng8OV51VvqjCqs1LalDdNCOD+66ZxNqCcrbpmhEBo7EtyGsQPQiqwf9hIRb+78NTKKtt5pG3D/HoO4e59jcbOFxhJTYilGe2HO0sa7cbiqs0QajB96n5mUSEWnh9T0/Tmil/09hqI8rPlxbtyYUkiKCr484bO5xbZmbw+PvF/HLtIa7OTePtry/ko7NHs7agnGprKwBltc20tNs1QahBFx0RyqKJKbyx75Q2MwWIxmBtYhKRBhGp9/BowHFPRND5zrWTuH7aSB7/1Cwe+8RMUmIj+Pic0bTbDC/tLAPOzMGkN8kpX7hu2igqGlq1mSlABG0ntTEm1hgT5+ERa4wJzCvuw/CYCH5360yW5I3o3DdxRCzTRyfwr63HMcZ0zuKqN8kpX7hyUqo2MwWQodpJPaQsnzOawxVWdhyrpajSSkJUGEnR4b4OSw1B2swUOIwxNLbZgv4+iCHv+otGERUewnNbj1PoHMGkC7koX9FmpljiXMAAABoZSURBVMDQ2mHHZjfB2cSkzoiJCOX6aSN5dc8JDp5qYLx2UCsf0mamwGAN4In6QBPEOfn4nDE0tdmoa25nXKr2Pyjf0WamwNAYwFN9gyaIczJzTELnyCUd4qp8TZuZ/J81gGdyBU0Q50REuHXuGEQcI5uU8iVtZvJ/jZ3LjWqCGBJuvySLNV9bQEZilK9DUUOcq5lp9b5TtNvObT0TNTjOrAWho5iGBItFyEnT2oPyDx+fM5rKhlZW7dBV5/yRdlIrpXzmiompXJQRz6PrCs95VUTlfYG8mhxoglAqoIkIX7tqAqWnm3lRaxF+RzuplVI+tWhiCtNHJ/A7rUX4HVcndfQQnM1VKeUHHLWIHMpqm3lhu9Yi/EljWweRYRZCQwLzT21gRq2UOsvCCY5axGPvai3CnwTyRH2gCUKpoCAi3HP1BMpqm3lu23Ffh6OcAnmqb9AEoVTQWJCTzKzMRP74XhHG6PQb/qCxtSNgp9kATRBKBQ0R4eOzR1N6upn9Jxt8HY5Cm5iUUn5k0aQUAN49WOHjSBQ4RjEF6l3U4OUEISLLROSgiBSKyH0ejj8iIrucj0MiUuvcP11ENolIvojsEZGPezNOpYJFamwkU9PjWXdAE4Q/0D6IHohICPAYcA2QC6wQkVz3MsaYe4wx040x04FHgVXOQ03Ap40xecAy4NcikuCtWJUKJldMSmXnsdOcbmzzdShDnjYx9WwuUGiMKTbGtAErgRt7Kb8CeBbAGHPIGHPY+fwEUAGkeDFWpYLG4kmp2A28d6jS16EMeVqD6Fk64D7ertS5rxsRyQSygXUejs0FwoEiD8fuEJFtIrKtslJ/GZQCmJYeT3JMuDYz+Zjd7lqPWhOEJ54WbO5p7N1y4AVjjO2sFxAZCfwd+IwxptvdP8aYx40xs40xs1NStIKhFDhmHF40MZX3DlXSodOA+0xTu2stCO2k9qQUGO22nQH0tLLJcpzNSy4iEge8DnzPGLPZKxEqFaQWT0qlrrmdncdrfR3KkBXoM7mCdxPEViBHRLJFJBxHEnilayERmQgkApvc9oUDLwFPG2Oe92KMSgWly3KSCbWINjP5kDXA16MGLyYIY0wHcCfwFrAfeM4Yky8iD4nIDW5FVwArzdm3fn4MWADc7jYMdrq3YlUq2MRFhjEnK4l3NUH4TDDUILwauTFmNbC6y74Humw/6OG8fwD/8GZsSgW7xZNS+eHq/ZTVNpOeMMzX4Qw51gBfbhT0TmqlgtYVk1IBtBbhI661IAL5PojAjVwp1atxKdGMSYriiQ+OUGVtJXdkHLmj4khPGIaIp0GGaiBpE5NSym+JCHcuHs+f3iviN+8cxtXLt3zOaH5yyzTfBjcEuJqYtAahlPJLH5s9mo/NHk1TWwcHTjXwqzWHWFtQzo9vNlqL8LKmtsCvQWgfhFJDQFR4KDPHJHLt1JFUN7ZRUt3k65CCntXZBxEVpp3USqkAMDsrEYBtJTU+jiT4ORYLCsFiCdyamiYIpYaQ8SkxxEWGsv3oaV+HEvQCfaI+0ASh1JBisQizMhPZpgnC6wJ9qm/QBKHUkDM7K4nCCiu1TbpehDdpDUIpFXBmZTr6IbSZybsCfblR0ASh1JBzUUYCoRbRZiYv0yYmpVTAGRYewpT0eLaXaILwpsY2bWJSSgWg2ZmJ7C6tpa1DFxTyFu2DUEoFpNlZibR22Nl3os7XoQQtbWJSSgWkWZlJANrMNEAef7/orJsPO2x2WtrtAb1YEGiCUGpISomNIHN4FNuO6h3VF8ra2sGP3zjAUxtLOvc1tjmm2dBRTEqpgDQrM5HtR09z9mKO6lzll9VhDOwtO9Nc1xgEM7mCJgilhqzZmUlUWds4qhP3XRBXYjhW00RdUzsQHGtBgCYIpYaszon79H6IC7LPreaQ7+z0D4a1IEAThFJD1viUGBKiwnhu63FaO2y+Didg7S2r67w73VWbcC03qjUIpVRAsliE/70uly0lNdz1zE46bHpPxLmytnZQXNXI5TnJpCcM60wQ1s4mJu2kVkoFqFtmZfDgh3JZU1DON1/Yg92uHdbnouBEPcbA1PR4pqTHkX+iHgieTurAjl4pdcFuvzSbxjYbP3/rINERITx84xRdjrSfXDWGqenx7D9Zz1v55dS3tNMYBMuNgiYIpRTw5UXjaGjp4I/vFZGTGsttl2T16zxjhvba1vvK6kiNjSA1LpIp6fEA5JfVaye1Uip4iAjfXjaRWZmJPL2ppM97I043tnHfi3uY/MCbHDhVPzhB+qG9ZXVMdSaGzgRxoo7G1g4sAhGhgf0nNrCjV0oNGBHho7MyKKpsZHep5zma7HbDc1uPs/iX63l+eyltHXZe3X1ikCP1D42tHRRVWjsTQ3JMBCPjI9lbVudcCyI04GtXmiCUUp2unTaSiFALL2w/3u2YtbWD5X/ezLde3MO4lBhev/sy5o8dzpr8ch9E6nsFJ890ULtMSY9nb1ldUEzUB5oglFJu4iLDWDZlBK/uPtnt3og/ri9iy5EafnTTVJ774sVMGhHHktw0DldYKa60+ihi39nrrGVNzXBLEKPiOVLVSEVDa8B3UIOXE4SILBORgyJSKCL3eTj+iIjscj4OiUit27HbROSw83GbN+NUSp1xy8wM6prbeWd/Ree+8voW/vJBMddPG8mt88ZgsTiaTq7OGwHA2oKhV4vYV1ZHSmwEaXGRnfumZsRhDOw4eloTRG9EJAR4DLgGyAVWiEiuexljzD3GmOnGmOnAo8Aq57lJwPeBecBc4PsikuitWJVSZ1w6PpkRcZG8uL20c9+v3z6EzW745tKJZ5VNTxjGlPQ41gzBBOHeQe3i6o9wNDEF9k1y4N0axFyg0BhTbIxpA1YCN/ZSfgXwrPP5UmCtMabGGHMaWAss82KsSimnEItw08x01h+qpLKhlcKKBv619TifmJdJ5vDobuWX5I5gx7HTVDS0+CBa32hqO7uD2iU1NpK0uAiAgF8LArybINIB956uUue+bkQkE8gG1p3LuSJyh4hsE5FtlZWVAxK0UsrRzGSzG17eVcZP3zxIVHgody0e77Hskrw0jOGsJqlgV3CiHnuXDmqXKaMc+7STuneexnf1NLh6OfCCMcbVK9avc40xjxtjZhtjZqekpJxnmEqprsanxjB9dAJ/fK+ItQXlfHHBWIbHRHgsOzEtljFJUazJPzXIUfqO+x3UXblqFdoH0btSYLTbdgbQ04Dp5ZxpXjrXc5VSXnDLrAyqrG2kxkbwucuzeywnIizJTWNjYXXnHcTBbm9ZHckxEZ3NSe6maoLol61Ajohki0g4jiTwStdCIjIRSAQ2ue1+C1giIonOzuklzn1KqUFyw7RRpCcM4/5rJxHVR3v6krwRtNnsvHdwaDT15pfVMzU9zuONcK5hr7GRgZ8gvHYFxpgOEbkTxx/2EOBJY0y+iDwEbDPGuJLFCmClcbu33xhTIyIP40gyAA8ZY3TxXKUGUXxUGBvvW9yvsrMyE0mKDmdNwSmumzbSYxmb3WBt6SA+Kmwgwxx0rR02iiqtXJWb6vF4Wlwkj906kznZgT/w0qspzhizGljdZd8DXbYf7OHcJ4EnvRacUmrAhFiEqyan8sbeU7S024gMO3uIpzGGL/9zO1tLTvPuNxYFdJIorLDSYTdMHhnXY5mekmSg0TuplVID4uaZGTS0dvDD1/d3O/bijjLeyi+nprGNJz4o9kF0A+fAyQYAJo3oOUEEC00QSqkBMX/scO5YMJa/bz7KK24T+J2obeYHr+YzNyuJpXlpPLmxhNqmNh9GemH2n6wnItRC1vAoX4fidZoglFID5ptLHVOG3//iHooqrRhj+PaLe7DZDT//6DTuuXoC1tYO/rzBN7WI5jYbL24v5WN/2sRlP11HXXP7Ob/GgVMNTEiLJTQk+P98Bv8VKqUGTViIhd/dOoOIsBC+/I8dPPHBETYcruI7104mc3g0k0bEcd20kfx1Ywk1jWfXIvaV1fHq7hPsc86GOpBO1bXwnZf2MveHb/ON53dzoraZ0tPNPL+t+6y1fTlwqp7JI2MHND5/FfjjsJRSfmVk/DB+/fHp3PbUFv7v9f1cnpPMJ+aN6Tz+tStzWL33JI+/X8x910zCbjf8fn0hv1p7CPclsVNjI8hKjiZreBRZydFkD49m/tjhJEaHn1M8q/ee5P5Ve2lpt3Hd1JF8bM5o5mUn8ZE/buLvm4/y2UuzOycf7EtlQytV1rYh0f8AmiCUUl6wYEIK9y6ZyD82H+Wnt0w7636BnLRYPjRtFE9vKuHmmek8/FoBGw5XccNFo7hjwVhKTzdRXNXIkcpGSqobWXegkiqrY+LA8FAL108byacvzmL66IReY7C2dvDgK/m8sL2UizLi+fXyGWQnn5lL6rZLsrj72Z28d6iSKyZ5HrLa1f6TjtXzJmkNQimlzt9XrhjPlxaO8/jt/O4rc3htzwmu+c0GQizCj2+eyvI5oxGRbhPgATS0tHO4wsqqHaW8tKOMVTvKmJYRz51XjOfq3LRuN6y9e7CC77+cT+npJu5ePJ67rswhrEufwbK8EaTGRvDX/5T0O0G4lledrDUIpZS6MD013YxPjeGT8zPZcqSGX31sOrmjev+DGxsZxswxicwck8i3l03i3zvLeGpjCXf8fTvzspP43nW5TM1wLNbzf68V8M6BCrKTo3nuixczOyvJ42uGh1q4dd4Yfv32YY5UNZ5Vu+jJgZMNjIiLPOdmrkAlfS1OHihmz55ttm3b5uswlFKDpMNm59mtx/n12kNUN7Zx2fhk/nukmojQEO6+cjy3X5JNeGjv43AqGlq49Cfr+OT8TL7/obw+3/Oa32wgLS6Cv35m7kBdhs+JyHZjzGxPx3QUk1IqIIWGWPjU/Eze/eYivrRoHPkn6rhxejrr7l3IHQvG9ZkcwLF+w7VTR/LCtlIa+xg51dZhp7CiYch0UIMmCKVUgIuLDOPbyyax84El/OKjF5EaG9n3SW4+fXEWDa0drNpR2mu54ior7TYzZIa4giYIpdQQN3NMAlPT4/nbpqP01uTummKjtzmYgo0mCKXUkCYifGp+JoUVVrYfPd1juf2n6gkPsfSrMztYaIJQSg15100bSUxEKCu39nxn9f6TDYxPjek2XDaYDZ0rVUqpHkRHhPKhi0bx+p6T1Ld4np/pwMn6IXODnIsmCKWUApbPGU1zu41Xd3df3bja2kpFQyu5Q6j/ATRBKKUUANMy4pk0IpZ/eWhmOnBq6KwB4U4ThFJK4eisXj5nNHtK68g/UXfWsaE2B5OLJgillHL68Ix0wkMtPNelFrGntI7kmAiSYyJ8FJlvaIJQSimnhKhwrpkygpd2ltHSbqOuqZ2vrtzJK7tPsHBCiq/DG3Q6WZ9SSrn5+JzRvLzrBD9avZ81+eVUWlu556oJfPmKcb4ObdBpglBKKTfzs4eTOTyKpzcdZVxKNI9/+hKmZfS+9kSw0gShlFJuLBbhoRunsOd4LV9YMJbIsBBfh+QzmiCUUqqLhRNShmSfQ1faSa2UUsojTRBKKaU80gShlFLKI00QSimlPPJqghCRZSJyUEQKReS+Hsp8TEQKRCRfRJ5x2/8z5779IvJbEfG8+rlSSimv8NooJhEJAR4DrgZKga0i8ooxpsCtTA5wP3CpMea0iKQ6918CXApMcxb9AFgIrPdWvEoppc7mzRrEXKDQGFNsjGkDVgI3dinzBeAxY8xpAGNMhXO/ASKBcCACCAPKvRirUkqpLryZINIB9xmvSp373E0AJojIRhHZLCLLAIwxm4B3gZPOx1vGmP1d30BE7hCRbSKyrbKy0isXoZRSQ5U3b5Tz1GfQdUXwUCAHWARkABtEZAqQDEx27gNYKyILjDHvn/VixjwOPA4gIpUictTDe8YDdX3sc9/29Nz1bzJQ5eE9+sNTHP0t09c19HQ9nsp48xp6O97b/3nX7b6e++IaBuJz5P78fK/Bm5+jrtu9/S6Af15Df67H336f+7vtrd+FzB5LGGO88gAuxvHN37V9P3B/lzJ/BG53234HmAN8E/hft/0PAN86zzge72uf+7an527/bruA/49ucfS3TF/X0NP19HAtXruG3o739n/en5+Br69hID5HA3EN3vwc9TNu931+dw39uR5/+33u7/Zg/y4YY7zaxLQVyBGRbBEJB5YDr3Qp82/gCgARScbR5FQMHAMWikioiITh6KDu1sTUT6/2Y9+rfTz39BoDEUd/y/R1DT1dT29lzkdfr9Hb8d7+z7tu9+f5+TrfaxiIz1F/3r8v3vwcdd0Opt8F9+f+dg393R7s3wXEmUm8QkSuBX4NhABPGmN+KCIP4ch6rziHrv4SWAbYgB8aY1Y6R0D9HliAo1nqTWPM170WaD+JyDZjzGxfx3Eh9Br8g16D7wV6/OD9a/DqZH3GmNXA6i77HnB7boCvOx/uZWzAF70Z23l63NcBDAC9Bv+g1+B7gR4/ePkavFqDUEopFbh0qg2llFIeaYJQSinl0ZBNECLypIhUiMi+8zh3lojsdc4xddY8USJyl3P+qXwR+dnARt0tjgG/BhF5UETKRGSX83HtwEd+Vhxe+Tk4j98rIsY5Qs5rvPRzeFhE9jh/BmtEZNTAR94Zgzfi/7mIHHBew0si4tU1O710DR91/h7bRcRrHcEXEnsPr3ebiBx2Pm5z29/r74tH5zuGNtAfOEZIzQT2nce5W3Dc5yHAG8A1zv1XAG8DEc7t1AC8hgeBewP55+A8Nhp4CzgKJAfaNQBxbmXuBv4YYPEvAUKdz38K/DQAfwaTgYk45oCb7W+xO+PK6rIvCcetAklAovN5Ym/X2dtjyNYgjOOu7Br3fSIyTkTeFJHtIrJBRCZ1PU9ERuL45d1kHP/rTwMfdh7+EvATY0yr8z0qup4fANcwqLx4DY8A36L73fsDzhvXYIypdysajRevw0vxrzHGdDiLbubMrAiBdA37jTEHvRn3hcTeg6XAWmNMjXHMcbcWWHa+v/NDNkH04HHgLmPMLOBeHPdidJWOY14pF/c5piYAl4vIf0XkPRGZ49VoPbvQawC409k08KSIJHov1B5d0DWIyA1AmTFmt7cD7cUF/xxE5Icichz4BI7ZBAbTQHyOXD6L4xvrYBvIaxhs/Yndk57mwDuv6/TqfRCBRERigEuA592a5iI8FfWwz/XtLhRHtW4+jilDnhORsc6M7XUDdA1/AB52bj+M40bGzw5spD270GsQkSjguziaOHxigH4OGGO+C3xXRO4H7gS+P8ChejRQ8Ttf67tAB/DPgYyxLwN5DYOtt9hF5DPAV537xgOrRaQNOGKMuYmer+e8rlMTxBkWoNYYM919pzju6t7u3HwFxx9Q9+pyBnDC+bwUWOVMCFtExI5jMq3Bmmr2gq/BGFPudt6fgde8GbAHF3oN44BsYLfzlysD2CEic40xp7wcu8tAfJbcPQO8ziAlCAYofmcH6fXAlYP1JcnNQP8MBpPH2AGMMU8BTwGIyHocc9mVuBUpxTH5qUsGjr6KUs7nOr3V8RIIDyALt44h4D/AR53PBbioh/O24qgluDp7rnXu/x/gIefzCTiqehJg1zDSrcw9wMpA+zl0KVOClzupvfRzyHErcxfwQoDFvwwoAFK8/X/v7c8RXu6kPt/Y6bmT+giOloxE5/Ok/lynx7gG64fnbw/gWRxrTbTjyK6fw/HN801gt/PD/UAP584G9gFFwO84c0d6OPAP57EdwOIAvIa/A3uBPTi+YY0MtGvoUqYE749i8sbP4UXn/j04JlVLD7D4C3F8QdrlfHhtFJYXr+Em52u14liw7C1/ih0PCcK5/7PO//9C4DPn8vvS9aFTbSillPJIRzEppZTySBOEUkopjzRBKKWU8kgThFJKKY80QSillPJIE4QKaiJiHeT3+4uI5A7Qa9nEMZvrPhF5ta8ZUUUkQUS+PBDvrRToinIqyImI1RgTM4CvF2rOTELnVe6xi8jfgEPGmB/2Uj4LeM0YM2Uw4lPBT2sQasgRkRQReVFEtjoflzr3zxWR/4jITue/E537bxeR50XkVWCNiCwSkfUi8oI41jz4p2tufef+2c7nVueEe7tFZLOIpDn3j3NubxWRh/pZy9nEmckIY0TkHRHZIY75/W90lvkJMM5Z6/i5s+w3ne+zR0R+MID/jWoI0AShhqLfAI8YY+YAtwB/ce4/ACwwxszAMXvqj9zOuRi4zRiz2Lk9A/gakAuMBS718D7RwGZjzEXA+8AX3N7/N87373M+HOf8QVfiuLMdoAW4yRgzE8caJL90Jqj7gCJjzHRjzDdFZAmQA8wFpgOzRGRBX++nlItO1qeGoquAXLeZMuNEJBaIB/4mIjk4ZroMcztnrTHGfc7+LcaYUgAR2YVjLp0PurxPG2cmO9wOXO18fjFn5uJ/BvhFD3EOc3vt7Tjm9gfHXDo/cv6xt+OoWaR5OH+J87HTuR2DI2G838P7KXUWTRBqKLIAFxtjmt13isijwLvGmJuc7fnr3Q43dnmNVrfnNjz/LrWbM518PZXpTbMxZrqIxONINF8BfotjfYgUYJYxpl1ESoBID+cL8GNjzJ/O8X2VArSJSQ1Na3CsrwCAiLimVY4HypzPb/fi+2/G0bQFsLyvwsaYOhzLjt4rImE44qxwJocrgExn0QYg1u3Ut4DPOtcXQETSRSR1gK5BDQGaIFSwixKRUrfH13H8sZ3t7LgtwDFNO8DPgB+LyEYgxIsxfQ34uohsAUYCdX2dYIzZiWNmz+U4Ft+ZLSLbcNQmDjjLVAMbncNif26MWYOjCWuTiOwFXuDsBKJUr3SYq1KDzLnqXbMxxojIcmCFMebGvs5TarBpH4RSg28W8DvnyKNaBnFJV6XOhdYglFJKeaR9EEoppTzSBKGUUsojTRBKKaU80gShlFLKI00QSimlPPp/WMrhneObOsgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.451934</td>\n",
       "      <td>0.357454</td>\n",
       "      <td>0.851360</td>\n",
       "      <td>24:01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Save: No need to run this if you want to train with your own data\n",
    "learn.save('first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('first');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I'm not going to say unfreeze. \n",
    "# Instead, I'm going to say freeze_to. \n",
    "# What that says is unfreeze the last two layers, don't unfreeze the whole thing. \n",
    "# We've just found it really helps with these text classification not to unfreeze the whole thing, \n",
    "# but to unfreeze one layer at a time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. unfreeze the last two layers\n",
    "# 2. train it a little bit more\n",
    "# 3. unfreeze the next layer again\n",
    "# 4. train it a little bit more\n",
    "# 5. unfreeze the whole thing\n",
    "# 6. train it a little bit more"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.348551</td>\n",
       "      <td>0.236512</td>\n",
       "      <td>0.907800</td>\n",
       "      <td>25:45</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.freeze_to(-2)\n",
    "learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('second')\n",
    "learn.load('second');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1250', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1250 00:00<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 92.00 MiB (GPU 0; 4.00 GiB total capacity; 2.85 GiB already allocated; 21.47 MiB free; 58.27 MiB cached)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-30-644a31004af3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfreeze_to\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mslice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m5e-3\u001b[0m\u001b[1;33m/\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2.6\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m5e-3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmoms\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0.8\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0.7\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[1;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)\u001b[0m\n\u001b[0;32m     21\u001b[0m     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,\n\u001b[0;32m     22\u001b[0m                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))\n\u001b[1;32m---> 23\u001b[1;33m     \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr,  moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[0;32m    198\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mwd\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    199\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 200\u001b[1;33m         \u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    201\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m->\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[0;32m     99\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0myb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 101\u001b[1;33m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    102\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    103\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[1;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mxb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0myb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0myb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0myb\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 26\u001b[1;33m     \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     27\u001b[0m     \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_loss_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\container.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m     90\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 92\u001b[1;33m             \u001b[0minput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     93\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\learner.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    263\u001b[0m         \u001b[0mraw_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mmasks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    264\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 265\u001b[1;33m             \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    266\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m>\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msl\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    267\u001b[0m                 \u001b[0mmasks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpad_idx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\models\\awd_lstm.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, from_embeddings)\u001b[0m\n\u001b[0;32m    107\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbs\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    108\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 109\u001b[1;33m         \u001b[0mraw_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minput_dp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mfrom_embeddings\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoder_dp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    110\u001b[0m         \u001b[0mnew_hidden\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mraw_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    111\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhid_dp\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhidden_dps\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\models\\awd_lstm.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, words, scale)\u001b[0m\n\u001b[0;32m     67\u001b[0m             \u001b[0msize\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m             \u001b[0mmask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdropout_mask\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membed_p\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 69\u001b[1;33m             \u001b[0mmasked_embed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     70\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmasked_embed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     71\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mscale\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmasked_embed\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscale\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 92.00 MiB (GPU 0; 4.00 GiB total capacity; 2.85 GiB already allocated; 21.47 MiB free; 58.27 MiB cached)"
     ]
    }
   ],
   "source": [
    "learn.freeze_to(-3)\n",
    "learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('third')\n",
    "learn.load('third');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='2', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/2 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1250', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1250 00:00<00:00]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 92.00 MiB (GPU 0; 4.00 GiB total capacity; 2.83 GiB already allocated; 19.47 MiB free; 79.78 MiB cached)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-31-8c22c7d70798>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munfreeze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mslice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1e-3\u001b[0m\u001b[1;33m/\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2.6\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1e-3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmoms\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0.8\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0.7\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[1;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)\u001b[0m\n\u001b[0;32m     21\u001b[0m     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,\n\u001b[0;32m     22\u001b[0m                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))\n\u001b[1;32m---> 23\u001b[1;33m     \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr,  moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[0;32m    198\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mwd\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    199\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 200\u001b[1;33m         \u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    201\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m->\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[0;32m     99\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0myb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 101\u001b[1;33m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    102\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    103\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[1;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mxb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0myb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0myb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0myb\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 26\u001b[1;33m     \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     27\u001b[0m     \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_loss_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\container.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m     90\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 92\u001b[1;33m             \u001b[0minput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     93\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\learner.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    263\u001b[0m         \u001b[0mraw_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mmasks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    264\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 265\u001b[1;33m             \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    266\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m>\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msl\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    267\u001b[0m                 \u001b[0mmasks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpad_idx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\models\\awd_lstm.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, from_embeddings)\u001b[0m\n\u001b[0;32m    107\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbs\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    108\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 109\u001b[1;33m         \u001b[0mraw_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minput_dp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mfrom_embeddings\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoder_dp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    110\u001b[0m         \u001b[0mnew_hidden\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mraw_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    111\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhid_dp\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhidden_dps\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\models\\awd_lstm.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, words, scale)\u001b[0m\n\u001b[0;32m     67\u001b[0m             \u001b[0msize\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m             \u001b[0mmask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdropout_mask\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msize\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membed_p\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 69\u001b[1;33m             \u001b[0mmasked_embed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     70\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmasked_embed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0memb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     71\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mscale\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmasked_embed\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscale\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 92.00 MiB (GPU 0; 4.00 GiB total capacity; 2.83 GiB already allocated; 19.47 MiB free; 79.78 MiB cached)"
     ]
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\bokhy\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\torch_core.py:83: UserWarning: Tensor is int32: upgrading to int64; for better performance use int64 input\n",
      "  warn('Tensor is int32: upgrading to int64; for better performance use int64 input')\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 28.00 MiB (GPU 0; 4.00 GiB total capacity; 2.83 GiB already allocated; 19.47 MiB free; 80.10 MiB cached)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-32-7b4d0a1b41b5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"I really loved that movie, it was awesome!\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, item, return_x, batch_first, with_dropout, **kwargs)\u001b[0m\n\u001b[0;32m    371\u001b[0m         \u001b[1;34m\"Return predicted class, label and probabilities for `item`.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    372\u001b[0m         \u001b[0mbatch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mone_item\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 373\u001b[1;33m         \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpred_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwith_dropout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mwith_dropout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    374\u001b[0m         \u001b[0mraw_pred\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgrab_idx\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mres\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbatch_first\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_first\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    375\u001b[0m         \u001b[0mnorm\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'norm'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mpred_batch\u001b[1;34m(self, ds_type, batch, reconstruct, with_dropout, activ)\u001b[0m\n\u001b[0;32m    350\u001b[0m         \u001b[0mactiv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mifnone\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mactiv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_loss_func2activ\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    351\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 352\u001b[1;33m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mwith_dropout\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mpreds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcb_handler\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    353\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mpreds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_dropout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcb_handler\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    354\u001b[0m             \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mactiv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[1;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mxb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0myb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0myb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0myb\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 26\u001b[1;33m     \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     27\u001b[0m     \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_loss_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\container.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m     90\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 92\u001b[1;33m             \u001b[0minput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     93\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\learner.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    263\u001b[0m         \u001b[0mraw_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mmasks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    264\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 265\u001b[1;33m             \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    266\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m>\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msl\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_len\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    267\u001b[0m                 \u001b[0mmasks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbptt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpad_idx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\models\\awd_lstm.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, from_embeddings)\u001b[0m\n\u001b[0;32m    110\u001b[0m         \u001b[0mnew_hidden\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mraw_outputs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    111\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mrnn\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mhid_dp\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrnns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhidden_dps\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 112\u001b[1;33m             \u001b[0mraw_output\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_h\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrnn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mraw_output\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhidden\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0ml\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    113\u001b[0m             \u001b[0mnew_hidden\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_h\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    114\u001b[0m             \u001b[0mraw_outputs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mraw_output\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\text\\models\\awd_lstm.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m     47\u001b[0m             \u001b[1;31m#To avoid the warning that comes because the weights aren't flattened.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     48\u001b[0m             \u001b[0mwarnings\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msimplefilter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"ignore\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 49\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodule\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     50\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     51\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\rnn.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, hx)\u001b[0m\n\u001b[0;32m    562\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward_packed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    563\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 564\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    565\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    566\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\rnn.py\u001b[0m in \u001b[0;36mforward_tensor\u001b[1;34m(self, input, hx)\u001b[0m\n\u001b[0;32m    541\u001b[0m         \u001b[0munsorted_indices\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    542\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 543\u001b[1;33m         \u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward_impl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_batch_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msorted_indices\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    544\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    545\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpermute_hidden\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munsorted_indices\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\torch\\nn\\modules\\rnn.py\u001b[0m in \u001b[0;36mforward_impl\u001b[1;34m(self, input, hx, batch_sizes, max_batch_size, sorted_indices)\u001b[0m\n\u001b[0;32m    524\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mbatch_sizes\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    525\u001b[0m             result = _VF.lstm(input, hx, self._get_flat_weights(), self.bias, self.num_layers,\n\u001b[1;32m--> 526\u001b[1;33m                               self.dropout, self.training, self.bidirectional, self.batch_first)\n\u001b[0m\u001b[0;32m    527\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    528\u001b[0m             result = _VF.lstm(input, batch_sizes, hx, self._get_flat_weights(), self.bias,\n",
      "\u001b[1;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 28.00 MiB (GPU 0; 4.00 GiB total capacity; 2.83 GiB already allocated; 19.47 MiB free; 80.10 MiB cached)"
     ]
    }
   ],
   "source": [
    "learn.predict(\"I really loved that movie, it was awesome!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Classification Interpretation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\bokhy\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\torch_core.py:83: UserWarning: Tensor is int32: upgrading to int64; for better performance use int64 input\n",
      "  warn('Tensor is int32: upgrading to int64; for better performance use int64 input')\n",
      "..\\aten\\src\\ATen\\native\\cuda\\LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n",
      "..\\aten\\src\\ATen\\native\\cuda\\LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n",
      "..\\aten\\src\\ATen\\native\\cuda\\LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n",
      "..\\aten\\src\\ATen\\native\\cuda\\LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span style=\"font-family: monospace;\"><span title=\"0.298\" style=\"background-color: rgba(206, 206, 229, 0.5);\">xxbos</span> <span title=\"0.292\" style=\"background-color: rgba(208, 208, 230, 0.5);\">xxmaj</span> <span title=\"1.000\" style=\"background-color: rgba(63, 0, 125, 0.5);\">zombiegeddon</span> <span title=\"0.298\" style=\"background-color: rgba(206, 206, 229, 0.5);\">was</span> <span title=\"0.438\" style=\"background-color: rgba(172, 171, 209, 0.5);\">perhaps</span> <span title=\"0.188\" style=\"background-color: rgba(228, 227, 239, 0.5);\">the</span> <span title=\"0.240\" style=\"background-color: rgba(219, 219, 235, 0.5);\">xxup</span> <span title=\"0.429\" style=\"background-color: rgba(175, 174, 211, 0.5);\">greatest</span> <span title=\"0.384\" style=\"background-color: rgba(185, 186, 218, 0.5);\">movie</span> <span title=\"0.233\" style=\"background-color: rgba(221, 220, 236, 0.5);\">i</span> <span title=\"0.145\" style=\"background-color: rgba(235, 233, 243, 0.5);\">have</span> <span title=\"0.102\" style=\"background-color: rgba(241, 239, 246, 0.5);\">ever</span> <span title=\"0.071\" style=\"background-color: rgba(244, 243, 248, 0.5);\">seen</span> <span title=\"0.100\" style=\"background-color: rgba(241, 240, 246, 0.5);\">!</span></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.cm as cm\n",
    "\n",
    "txt_ci = TextClassificationInterpretation.from_learner(learn) # TextClassificationInterpretation can be used ONLY once we train a text classification model.\n",
    "test_text = \"Zombiegeddon was perhaps the GREATEST movie i have ever seen!\"\n",
    "txt_ci.show_intrinsic_attention(test_text,cmap=cm.Purples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\bokhy\\Anaconda3\\envs\\deeplearning\\lib\\site-packages\\fastai\\torch_core.py:83: UserWarning: Tensor is int32: upgrading to int64; for better performance use int64 input\n",
      "  warn('Tensor is int32: upgrading to int64; for better performance use int64 input')\n",
      "..\\aten\\src\\ATen\\native\\cuda\\LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n",
      "..\\aten\\src\\ATen\\native\\cuda\\LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n",
      "..\\aten\\src\\ATen\\native\\cuda\\LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n",
      "..\\aten\\src\\ATen\\native\\cuda\\LegacyDefinitions.cpp:19: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([0.2981, 0.2924, 1.0000, 0.2977, 0.4379, 0.1884, 0.2399, 0.4287, 0.3837,\n",
       "        0.2329, 0.1449, 0.1018, 0.0713, 0.1004], device='cuda:0')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "txt_ci.intrinsic_attention(test_text)[1] # Numeric interpretation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Text</th>\n",
       "      <th>Prediction</th>\n",
       "      <th>Actual</th>\n",
       "      <th>Loss</th>\n",
       "      <th>Probability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>xxbos i went to see this 3 nights ago here in xxmaj cork , xxmaj ireland . xxmaj it was the world premiere of it , in the tiny cinema in the xxmaj xxunk xxmaj arts xxmaj centre as part of the xxmaj cork xxmaj film xxmaj festival . \\n \\n  i found \" xxmaj strange xxmaj fruit \" to be an excellent movie . xxmaj it is a</td>\n",
       "      <td>pos</td>\n",
       "      <td>neg</td>\n",
       "      <td>8.45</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj this movie was pure genius . xxmaj john xxmaj waters is brilliant . xxmaj it is hilarious and i am not sick of it even after seeing it about 20 times since i bought it a few months ago . xxmaj the acting is great , although xxmaj ricki xxmaj lake could have been better . xxmaj and xxmaj johnny xxmaj depp is magnificent . xxmaj he is</td>\n",
       "      <td>pos</td>\n",
       "      <td>neg</td>\n",
       "      <td>7.91</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj this is definitely one of the best xxmaj kung fu movies in the history of xxmaj cinema . xxmaj the screenplay is really well done ( which is not often the case for this type of movies ) and you can see that xxmaj chuck ( in one of his first xxunk a great actor . xxmaj the final fight with the xxunk deputy in the bullring is</td>\n",
       "      <td>pos</td>\n",
       "      <td>neg</td>\n",
       "      <td>7.43</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos i really liked this quirky movie . xxmaj the characters are not the bland beautiful people that show up in so many movies and on xxup tv . xxmaj it has a realistic edge , with a captivating story line . xxmaj the main title sequence alone makes this movie fun to watch .</td>\n",
       "      <td>pos</td>\n",
       "      <td>neg</td>\n",
       "      <td>6.78</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj this has to be one of , if not xxup the greatest xxmaj mob / xxmaj crime films of all time . xxmaj every thing about this movie is great , the acting in this film is of true quality ; xxmaj master p 's acting skills make you actually believe he is xxmaj italian ! xxmaj the cinematography is excellent too , probably the best ever .</td>\n",
       "      <td>pos</td>\n",
       "      <td>neg</td>\n",
       "      <td>6.37</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "txt_ci.show_top_losses(5) # If max_len=None, it will display all tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}