Russian LM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train an AWD-LSTM language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *\n",
    "import os\n",
    "from tqdm import tqdm_notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path(\"artifacts\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.cuda.set_device(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The corpus is a subset of Taiga corpus, containing the following subcorpora (in folders of the same name):\n",
    "- Arzamas\n",
    "- Fontaka\n",
    "- Interfax\n",
    "- KP\n",
    "- Lenta\n",
    "- NPlus1\n",
    "\n",
    "Each folder contains the \"texts\" folder, containing texts, one per file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04d8fbb3930d406eac0c7c2fc22ef836",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "texts = list()\n",
    "for root, _, files in tqdm_notebook(os.walk(\"corpus/extracted\")):\n",
    "    for file in files:\n",
    "        with open(os.path.join(root, file)) as f:\n",
    "            texts.append(f.read().strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "208006138"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(len(t.split()) for t in texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = pd.DataFrame(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1181970, 1)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 64"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "RAM utilization peaks at around 40Gb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = TextList.from_df(texts, \n",
    "                        processor=[TokenizeProcessor(tokenizer=Tokenizer(lang=\"xx\")), \n",
    "                                                     NumericalizeProcessor(min_freq=2, max_vocab=60000)]).\\\n",
    "                random_split_by_pct(.1).\\\n",
    "                label_for_lm().\\\n",
    "                databunch(bs=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>  <col width='5%'>  <col width='95%'>  <tr>\n",
       "    <th>idx</th>\n",
       "    <th>text</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>0</th>\n",
       "    <th>риа xxmaj новости сообщил , что среди них не значатся йод и xxunk . \\n xxmaj правительство на прошлой неделе поручило xxmaj xxunk на время приостановить продажу xxunk жидкостей с содержанием спирта более 25 % . xxmaj речь идет о xxunk , парфюмерии и xxunk средств . \\n xxmaj вопрос о запрете оборота xxunk жидкостей возник после массового отравления в xxmaj иркутске « xxmaj xxunk » . xxmaj жертвами xxunk</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>xxunk - 01 - xxunk + 03:00;;http : / / www.kp.ru / online / news / xxunk / \\n xxmaj комсомольская xxunk два месяца назад молодого xxunk разыскивают в соседнем xxunk > xxup чп;;2017 - 01 - xxunk + 03:00;;http : / / www.kp.ru / online / news / xxunk / \\n xxmaj комсомольская xxunk xxmaj липецке рабочего xxunk xxunk xxunk xxunk > xxmaj криминал;;2017 - 01 - xxunk +</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>xxmaj жана xxmaj xxunk и xxmaj xxunk xxmaj xxunk , xxunk xxmaj xxunk xxmaj xxunk и xxmaj xxunk xxmaj xxunk , фотографиях xxmaj адольфа де xxmaj xxunk . xxmaj открытие выставки в музее xxunk приурочено к 50-летию кончины артиста , передает агентство xxup xxunk news . c начала 20-х годов xxmaj xxunk , звезда парижских \" xxmaj русских сезонов \" , находился под наблюдением врачей , а последние 20 лет</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>7 января \" xxmaj газпром \" приостановил и эти поставки , обвиняя xxmaj украину в незаконном отборе топлива и xxunk экспортных газопроводов . в свою очередь , президент xxmaj украины xxmaj виктор xxmaj ющенко потребовал от своего коллеги в xxmaj россии xxmaj дмитрия xxmaj медведева немедленно возобновить поставки . xxmaj пока этого не произошло , многие европейские страны xxunk большие объемы xxunk топлива . в некоторых странах , например в</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>в \" xxmaj газпроме \" пока не прокомментировали заявление иракских властей . в xxmaj ираке работает дочерняя компания газовой монополии - \" xxmaj газпром нефть \" . в 2009 году она выиграла тендер на разработку нефтяного месторождения xxmaj xxunk . xxmaj месторождение расположено на востоке xxmaj ирака . \" xxmaj газпром нефть \" является оператором проекта , компании принадлежит доля в 30 процентов . в августе 2012 года \" xxmaj</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.save(path / \"lm_data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data, pretrained_model=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt8VPWd//HXJ3dIwi0Xud8V76JEC1ap1kup66po29X9abVa6WW9VFe727rb7a+2a1vXVmtrK96rrW7Vdr3V+yp4gWpQQBTkHggiCQmQBJKQZD77x0xgjAkJkJlzJvN+Ph7zyDnfOTPfzwzMfOZ7vpdj7o6IiKSvjKADEBGRYCkRiIikOSUCEZE0p0QgIpLmlAhERNKcEoGISJpTIhARSXNKBCIiaU6JQEQkzWUFHUBPFBcX+9ixY4MOQ0QkpSxYsGCzu5d0d1xKJIKxY8dSXl4edBgiIinFzCp6clzCTg2Z2b1mVmVmS+LKfmhmG8xsYex2RqLqFxGRnklkH8H9wIxOyn/p7pNjt78msH4REemBhCUCd58L1Cbq+UVEpHcEMWroCjNbHDt1NDiA+kVEJE6yE8FvgQnAZGAjcEtXB5rZLDMrN7Py6urqZMUnIpJ2kpoI3H2Tu7e5ewS4CzhuD8fOdvcydy8rKel29JOIiOyjpCYCMxsWtzsTWNLVsSIikhwJm0dgZg8DJwHFZlYJ/AdwkplNBhxYC3wjUfWLiKSyTXVNPDS/gplHj2B8SUFC60pYInD3CzopvidR9YmI9CUVNTu4/X9X8plxRQlPBFprSEQkhOoaWwAY0C/xC0AoEYiIhFBdUywR5GUnvC4lAhGREGpvEQzsp0QgIpKWtjW2AlCYp1NDIiJpqa6phfycTLIyE/81rUQgIhJCdY0tDEjCaSFQIhARCaW6ppakdBSDEoGISChta2xJSkcxKBGIiIRSXWNrUuYQgBKBiEgo6dSQiEiaU2exiEgai0Sc+uZWJQIRkXRV39yKOwxIwmQyUCIQEQmd3QvOqUUgIpKWkrngHCgRiIiETl1snSENHxURSVPbkrjyKCgRiIiEjk4NiYikOXUWi4ikubqmVsygMFd9BCIiaamusYXC3CwyMiwp9SkRiIiETDKXlwAlAhGR0EnmgnOgRCAiEjrJXIIalAhEREKnz7QIzOxeM6sysyWd3HedmbmZFSeqfhGRVJXMq5NBYlsE9wMzOhaa2SjgNGBdAusWEUlZfaaz2N3nArWd3PVL4LuAJ6puEZFU1doWYfvOtr5xaqgzZnYWsMHdFyWzXhGRVFHflNwF5wCSVpOZ9QduAE7v4fGzgFkAo0ePTmBkIiLhkewF5yC5LYIJwDhgkZmtBUYC75jZ0M4OdvfZ7l7m7mUlJSVJDFNEJDjJXnAOktgicPf3gNL2/VgyKHP3zcmKQUQk7HZfi6APtAjM7GFgHjDJzCrN7LJE1SUi0lfsahH0hT4Cd7+gm/vHJqpuEZFUVdfH+whERKQb7Z3FfXb4qIiI7FldUwuZGUb/nMyk1alEICISInWNrQzIy8IsOdciACUCEZFQqWtK7vISoEQgIhIqyV5wDpQIRERCpa4xuUtQgxKBiEio1DUl96I0oEQgIhIqahGIiKQ59RGIiKSxppY2mlsjGjUkIpKudl2LIE99BCIiaWn3gnNqEYiIpKW6ANYZAiUCEZHQ2LXgnFoEIiLpqS7WRzBQ8whERNKTTg2JiKQ5dRaLiKS5bY0t5GRlkJedvGsRgBKBiEhoRK9FkNzWACgRiIiERvRaBMntKAYlAhGR0AhiwTlQIhARCY26xuRfnQyUCEREQqOuqTXpK4+CEoGISGhETw2pj0BEJC25eyAXrocEJgIzu9fMqsxsSVzZjWa22MwWmtkLZjY8UfWLiKSSppYILW3e5zqL7wdmdCi72d2PdPfJwNPADxJYv4hIyqjdsROAwf37UCJw97lAbYeyurjdfMATVb+ISCqprm8GoHRAbtLrTnqvhJn9BPgqsA04Odn1i4iEUVVdEwAlBXlJrzvpncXufoO7jwL+AFzR1XFmNsvMys2svLq6OnkBiogEoLoh2iIoKUx+iyDIUUN/BM7r6k53n+3uZe5eVlJSksSwRESSr/3UUFFBTtLrTmoiMLMD43bPApYls34RkbCqrm9mSH4O2ZnJ/32esD4CM3sYOAkoNrNK4D+AM8xsEhABKoBvJqp+EZFUUl3fTElB8k8LQQITgbtf0EnxPYmqT0QklVU3NAfSPwCaWSwiEgrV9UoEIiJpy92VCERE0ll9cyvNrZHA+giUCEREAtY+dFQtAhGRNKVEICKS5pQIRETS3K5EoD4CEZH0VN3QTHamMSiAJahBiUBEJHDts4rNLJD6lQhERAIW5BwCUCIQEQmcEoGISJoLcp0hUCIQEQlUW8SpaQhu5VFQIhARCVTt9p1EPLg5BKBEICISqKAnk4ESgYhIoIK8VnE7JQIRkQDtnlWcF1gMSgQiIgFqTwTFhcm/aH07JQIRkQBV1zdTkJtF/5yEXTm4W0oEIiIBqqpvCrR/AJQIREQC1b7OUJB6lAjMbIKZ5ca2TzKzq8xsUGJDExHp+4KeVQw9bxE8DrSZ2UTgHmAc8MeERSUikiaCXmcIep4IIu7eCswEbnX3a4BhiQtLRKTva2ppo76pNWUSQYuZXQBcDDwdKwvmCgoiIn1E0Fcma9fTRPA1YBrwE3dfY2bjgIcSF5aISN8XhlnF0MNE4O4fuPtV7v6wmQ0GCt39p3t6jJnda2ZVZrYkruxmM1tmZovN7C/qcBaRdBaGdYag56OGXjWzAWY2BFgE3Gdmv+jmYfcDMzqUvQgc7u5HAsuB7+1lvCIifUZ7IihNhUQADHT3OuBc4D53nwKcuqcHuPtcoLZD2QuxTmeA+cDIvYxXRKTPqK5vxgyG5Ae3vAT0PBFkmdkw4Cvs7izeX5cCz3Z1p5nNMrNyMyuvrq7upSpFRMKjuqGZovwcsjKDndvb09p/BDwPrHL3t81sPLBiXys1sxuAVuAPXR3j7rPdvczdy0pKSva1KhGR0Kqub6Y44BFDAD1a5cjdHwUejdtfDZy3LxWa2cXAmcAp7u778hwiIn1BGCaTQc87i0fGRvlUmdkmM3vczPb6/L6ZzQD+BTjL3Xfs7eNFRPqSlEoEwH3Ak8BwYATwVKysS2b2MDAPmGRmlWZ2GfBroBB40cwWmtnv9jlyEZEU1toWYVNdE0MHBHdBmnY9XQC7xN3jv/jvN7Pv7OkB7n5BJ8X39DgyEZE+bOO2Jlojzpii/kGH0uMWwWYzu9DMMmO3C4GaRAYmItKXVdREz46PHpIfcCQ9TwSXEh06+jGwEfgS0WUnRERkH1TUbgdInRaBu69z97PcvcTdS939HKKTy0REZB9U1OwgJysjFH0E+zOL4dpei0JEJM1U1Gxn1OB+ZGRY0KHsVyIIPnoRkRRVUbODMUXB9w/A/iUCTQYTEdkH7s662h2MHhJ8/wB0M3zUzOrp/AvfgH4JiUhEpI/b3LCTHTvbQtFRDN0kAncvTFYgIiLpYl2IRgzB/p0aEhGRfRCmOQSgRCAiknQVNTswg1FDwnGGXYlARCTJ1tXuYPjAfuRmZQYdCqBEICKSdBU120MzYgiUCEREkm5d7Y7QdBSDEoGISFI1NLeyuWEno5UIRETS07rYiKExIRkxBEoEIiJJFbY5BKBEICKSVLvmECgRiIikp4raHQzun82AvOygQ9lFiUBEJInW1exgdEhWHW2nRCAikkQVtdsZE6I5BKBEICKSNC1tET7a2hSqjmJQIhARSZoNWxppi3ioZhWDEoGISNJU1MbmEKiPQEQkPVXUhG8OASgRiIgkTUXNDvKyMygtzA06lE9IWCIws3vNrMrMlsSVfdnM3jeziJmVJapuEZEwqqiJXqfYzIIO5RMS2SK4H5jRoWwJcC4wN4H1ioiE0prNDaHrH4Burlm8P9x9rpmN7VC2FEhaNlxZVc/GbU379RzGnmPdm5ey16/aOu5+sqC9bqPr9zT+mPj99pKOzxF/nGGfeH3xZe3bGdb+HO3b0b8ZFj0uwyx2i96XmfHp7YzYdqYZGRnh+qUk0luaW9tYW7ODLx4+LOhQPiVhiSAMHnizggfnVwQdhuwFM8jKiCaGrIwMMjOM7EwjOzODrEwjJzODvOxMcrMyyM3KJD83i4LcTArysijMy2bYwDyGD+zH8EH9GDWkH4UhmsYv6W3N5u20RZwDDygIOpRPCW0iMLNZwCyA0aNH79NzfP3EcZw9efgnypye/zL37u7v7oBPHLsXB3dSd8eHe/sR3nWc7p88dvf+J2PyXc+z+zj3Tx7nux7vu+5zh8iu+6JHR9xj5dHtSCR6f/t2W2y/LRI9rs2jZZGI79puv7W0OW2RCC0Rp6U1QmvE2dkaobm1jebWCE0tbXy0tZGG5lYamlupb2qhpW33u5FhUDZmCKccUsqphx7AhJLwfQAlfXz4cT0Ak4YWBhzJp4U2Ebj7bGA2QFlZ2d59i8aMKcoP5fk4SYxIxNm8vZmNW5v4aGsjH2ys4+WlVdz07DJuenYZBx1QwCXHj+PcY0aQlx2Oa8VK+lixqYHMDGNccfi+k0KbCET2VkaGUVqYR2lhHkeNGsQXjxjGP58+iQ1bG3l56Sb+VL6e7//lPW5+fhkXTh3DRdPGUFqYF3TYkiY+3FTP2KL+oblgfbxEDh99GJgHTDKzSjO7zMxmmlklMA14xsyeT1T9Iu1GDOrHV6eN5akrTuCRWVMpGzuEX7+ykhN+9gr/9j/vsT4221MkkVZsqg/laSFI7KihC7q46y+JqlNkT8yMqeOLmDq+iDWbtzN77mr+9HYlD7+1nrOPGs63T57IxFL1I0jva2ppo6J2B2dPHhF0KJ3SzGJJS+OK87np3COY+92TueT4sTy75GNO/+Ucrv3Twl3LAIj0lpVVDbiHs6MYlAgkzQ0dmMe/n3kor//LyXz9xPE8s3gjn79lDv/6+GI2bG0MOjzpI5Zvio4YOiiEQ0dBiUAEgKKCXL5/xiG89t2TuWjqGP78zgZOvvlVfvjk+1TXNwcdnqS45ZsayM600I5iVCIQiVM6II8fnnUYr1x/EuceM4IH51cw/eev8PPnlrGtsSXo8CRFLd9Uz4SSArIzw/mVG86oRAI2YlA/fnrekbx07ec47dADuOPVVXzu5le4+7XVNLe2BR2epJjlm+o58IBw9g+AEoHIHo0rzudXFxzNM1edwBEjBvLjZ5Zyyi1zeGLhhr2eLS7paXtzK5VbGpkU0v4BUCIQ6ZHDhg/kwcs+w+8vPY7CvGyufmQh/3Dn/F2dgCJdWVHVAKAWgUhfMf2gEp658gR+eu4RLK+q54zbXuOmvy5le3Nr0KFJSO0eMaREINJnZGQY5x83mv/955M475iR3Dl3Naf9Yg6vLKsKOjQJoRWb6snNygjdBevjKRGI7KMh+Tn87EtH8tg3p5Gfm8XX7n+b7zzyLrXbdwYdmoTIh5samFhaQGaIr7WhRCCyn8rGDuHpq07g6lMO5Jn3NnLqL9SZLLut2FTPpBCfFgIlApFekZuVyTWnHcTTV57IqCH9ufqRhcx6cAFVdft3hTxJbdsaW9i4rSnUHcWgRCDSqyYNLeTP3zqe759xMHOXV3PqL+bwaPl6tQ7S1MqqcC8t0U6JQKSXZWYYs6ZP4NmrT2TS0EKuf2wxl/++nK071HeQbpZvig4dDfOIIVAiEEmY8SUF/PesafzgzEOZs7yav/vV6yxavzXosCSJPvy4nv45mYwY1C/oUPZIiUAkgTIyjEtPGMej3zwegC//bh6/n7dWp4rSxKLKrRw2fAAZIR4xBEoEIkkxedQgnr7yBD47sYgfPPE+Vz2ykAZNQuvTmlraWLJhG1PGDAk6lG4pEYgkyeD8HO65+Fiu/8Iknln8EWfd/jrLPq4LOixJkEXrt9LS5pSNGRx0KN1SIhBJoowM459OnsgfL59KfXMr5/zmDR4tXx90WJIA5RVbAJiiRCAinZk6vohnrjqBo0cN5vrHFnPtnxZqvaI+ZkHFFiaWFjA4PyfoULqlRCASkNLCPB76+me46pQD+Z93N3Dm7a+zZMO2oMOSXhCJOAsqtqTEaSFQIhAJVGaGce1pB/HHy6fSuLONmXe8wd2vrSYS0aiiVLayuoFtjS0pcVoIlAhEQmHq+CKevfpETppUyo+fWcqlD7ytayWnsPK10f6BY8eGf8QQKBGIhMbg/BxmXzSFG88+jDdX1fDF215jzvLqoMOSfVBeUUtxQQ5jisK79HQ8JQKREDEzLpo2liev+CxD8rO5+N63+PHTH7CzNRJ0aLIXytduYcqYwZiFeyJZu4QlAjO718yqzGxJXNkQM3vRzFbE/qbGCTSRJDt46ACevOIELpo6hrtfX8N5v32TtZu3Bx2W9EBVfRPrandQlgITydolskVwPzCjQ9m/Ai+7+4HAy7F9EelEXnYmN55zOL+7cArrandw5u2v88TCDUGHJd1YEOsfKBubOr9zE5YI3H0uUNuh+Gzggdj2A8A5iapfpK+YcfhQ/nr1iRw8tJCrH1nI9Y8u0pyDECuv2EJuVgaHDR8YdCg9luw+ggPcfSNA7G9pkusXSUkjBvXjkVlTueLkiTz2TiVn3q6VTMOqfG0tR40aRE5W6nTBhjZSM5tlZuVmVl5drZETIlmZGVz3hUk8fPlUmlraOO+3b/KbV1bSpjkHodG4s433P6pLmYlk7ZKdCDaZ2TCA2N+qrg5099nuXubuZSUlJUkLUCTspo4v4rmrp/OFw4Zy8/MfcsFd86moUUdyGCxcv5XWiKfM/IF2yU4ETwIXx7YvBp5Icv0ifcLA/tn8+h+P5uYvHcnSj+r4wq1zuef1NWodBOzNVZsxg2NGq0UAgJk9DMwDJplZpZldBvwUOM3MVgCnxfZFZB+YGV8uG8UL105n2vgibnz6A75y5zxWVTcEHVpaikScv7y7gRMmFjOwf3bQ4eyVRI4ausDdh7l7truPdPd73L3G3U9x9wNjfzuOKhKRvTRsYD/uveRYbvnyUazYVM8Zt73GnXNW0dqmSWjJNH9NDZVbGvnSlJFBh7LXQttZLCI9Z2acN2UkL137OT53UAk3PbuM8377Jh9+XB90aGnjsQWVFOZl8YXDhgYdyl5TIhDpQ0oH5HHnRVO4/YKjWb+lkTNvf41bX1pOc2tb0KH1aQ3NrTz73sf8/VHDycvODDqcvaZEINLHmBl/f9RwXrxmOl88fBi3vrSCM257jbfW6Exsovx18UYaW9pS8rQQKBGI9FlFBbn86oKjuf9rx9LcGuErd87je39ezNYdO4MOrc95bEEl40vyOXrUoKBD2SdKBCJ93EmTSnnhmunMmj6eP5VX8vlb5vDwW+s01LSXrN28nbfW1vKlKSNTZrXRjpQIRNJA/5wsvn/GITx95QlMLC3ge39+j5l3vMG767YEHVrKe/ydSjIMzj06NU8LgRKBSFo5ZNgA/nvWVG47fzKb6pqYecebXPfoIl0NbR+1RZzHF1Ry4oElDB2YF3Q4+0yJQCTNmBlnTx7By/98Et/43HieWLiBz//Xq9z92mpaNPdgr8xdXs1H25pStpO4nRKBSJoqyM3ie188hOe/M50pYwfz42eWMuPWubz0wSbc1X/Qnda2CDc9u5RRQ/px+mEHBB3OflEiEElz40sKuO+SY7n7q2W4w9d/X84/zJ6v/oNu/OFv61i+qYF/+7tDyc1KvbkD8ZQIRAQz49RDD+D5a6bz43MOZ3V1AzPveJNv/2EBq7V20ads2b6TX7y4nM9OLOL0Q1O7NQCQFXQAIhIe2ZkZXDh1DOccPYK75q7mrtdW8/z7m/hK2Si+c+qBHDAgdTtEe9MvX1pOQ3MrPzjzsJQdMhpPLQIR+ZSC3CyuOe0g5lx/MhdNHcNjC9bzuZtf4WfPLaOuqSXo8AK17OM6HppfwYWfGc2koYVBh9MrlAhEpEslhbn88KzDePnak5hx2FB+++oqTr75VR6ctzYtRxi5Oz966gMG9MvmmtMOCjqcXqNEICLdGl3Un1vPP5qnrohOSPv3J95nxq1zeW7JRiJpNEP5ntfX8OaqGq497SAG9c8JOpxeo0QgIj12xMiBPDJrKnd9tQwHvvnQO8y4bS5PLNzQ569/8NySj/nJX5dyxhFDufAzY4IOp1dZKowXLisr8/Ly8qDDEJE4rW0RnnlvI795ZSXLNzUwpqg/35g+gXOPGZGSSzHvycL1Wzl/9jwOGTaAhy+fmjKvz8wWuHtZt8cpEYjI/ohEnBeXbuI3r6xkceU2igtyuHjaWC6cOobB+al/+mR97Q5m3vEG/XIy+cu3P0txQW7QIfWYEoGIJJW7M291DbPnrubVD6vpl53JmUcO40tTRnLcuCEpOcyyqq6Jf7z7b1TXN/P4t45nYmlB0CHtlZ4mAs0jEJFeYWYcP6GY4ycU8+HH9dz7+hqeXvwRjy6oZPSQ/px3zEjOOXo4Y4rygw61R9Zs3s5F9/yN2u07ue+SY1MuCewNtQhEJGF27GzluSUf89iCSt5cVQPA5FGDOOuo4Zx55DBKQzpB7b3KbVxy31s4cP/XjuXIkal5wRmdGhKRUNmwtZGnF33Ek4s+4v2P6jCDI0cO4pSDS/n8waUcNnxAKE4fzVlezbcfWsCg/jk8eNlxjC9J3ZaAEoGIhNbKqgaefW8j//thFQvXb8UdigtymTp+CFPHFzF1fBETSvKTmhjeWbeF215awZzl1Rw8tJAHLj0u5ZfUUCIQkZSwuaGZVz+s5o2Vm5m3qoaP65oAGJKfw1EjBzJ51GCOGjWQI0YMpKiXR+w0tbQxb3UN976+htdWbGZIfg6Xnzier04bQ35u6nehKhGISMpxd9bV7mDeqhoWVGxhUeVWVlQ10P41VVqYyyHDBnDIsAFMKMlnbHE+Y4r6U1KQ223roS3iVNc3s3FbI0s2bOPVD6t5c1UNjS1tFOXnMGv6eC6c2jcSQLtQjxoys6uBywED7nL3W4OIQ0TCxcwYU5TPmKJ8zj9uNAD1TS28V7mN9z+qY+nGOj7YWMcbKzfTGre0Rb/sTPJzM8nOzCAnK4PszAzcHXdwor/8q+qbaYt7zOgh/flK2UhOOriUaeOLUmaSWCIkPRGY2eFEk8BxwE7gOTN7xt1XJDsWEQm/wrxsjp9YzPETi3eV7WyNsGFrI2trtrOuZgfra3ewo6WNltYIO9sitLRFMDMMyDAjJyuDoQPyGDowj+GD8hhXXMDYov6h6JwOgyBaBIcA8919B4CZzQFmAj8PIBYRSUE5WRmMK85nXHFqzEkIuyAWnVsCTDezIjPrD5wBjAogDhERIYAWgbsvNbOfAS8CDcAioLXjcWY2C5gFMHr06KTGKCKSTgJZhtrd73H3Y9x9OlALfKp/wN1nu3uZu5eVlJQkP0gRkTQR1KihUnevMrPRwLnAtCDiEBGR4Bade9zMioAW4J/cfUtAcYiIpL1AEoG7nxhEvSIi8mm6VKWISJpTIhARSXMpsdaQmW2jk5FFwEBgWw/3u9suBjbvQ3gd69ybY/pa/HuKM36/N+PfU3zd3d9d/B33O9tW/OGIH8LxGQjbZ3iMu3c/7DK6Hke4b8DsnpTvab+7baC8N2NLx/j3FGeHWHst/p68hn2Nv4fvu+IPQfz78xrS4TPc3S1VTg091cPyPe33ZHtf9OTx6RJ/x7KuXk9vxt+T59jX+Dvud7at+Pt+/Hs6JlU+w3uUEqeGksHMyr0Hy7WGleIPluIPXqq/hiDjT5UWQTLMDjqA/aT4g6X4g5fqryGw+NUiEBFJc2oRiIikuT6ZCMzsXjOrMrMl+/DYKWb2npmtNLNfWdyVK8zsSjP70MzeN7OEXT8hEfGb2Q/NbIOZLYzdzuj9yHfFkJD3P3b/dWbmZlbc1XPsrwS9/zea2eLYe/+CmQ3v/ch3xZCI+G82s2Wx1/AXMxvU+5HviiER8X859rmNmFlCzsPvT9xdPN/FZrYidrs4rnyPn5F9sr/DjsJ4A6YDxwBL9uGxbxFdBM+AZ4EvxspPBl4CcmP7pSkW/w+B61L1/Y/dNwp4HqgAilMpfmBA3DFXAb9LsfhPB7Ji2z8DfpZi8R8CTAJeBcrCFHcsprEdyoYAq2N/B8e2B+/pNe7PrU+2CNx9LtHlrXcxswlm9pyZLTCz18zs4I6PM7NhRD+w8zz6jv8eOCd297eAn7p7c6yOqhSLP2kSGP8vge8SvQxtwiQifnevizs0nwS+hgTF/4K7t183ZD4wMsXiX+ruHyYq5v2JuwtfAF5091qPLsr5IjAjUZ/xPpkIujAbuNLdpwDXAXd0cswIoDJuvzJWBnAQcKKZ/c3M5pjZsQmN9tP2N36AK2JN+3vNbHDiQu3UfsVvZmcBG9x9UaID7cJ+v/9m9hMzWw/8P+AHCYy1M73x/6fdpUR/iSZTb8afTD2JuzMjgPVx++2vJSGvMahlqJPKzAqA44FH406n5XZ2aCdl7b/csog20aYCxwJ/MrPxsaycUL0U/2+BG2P7NwK3EP1AJ9z+xm/RS5reQPT0RNL10vuPu98A3GBm3wOuAP6jl0PtVG/FH3uuG4heUfAPvRnjnvRm/Mm0p7jN7GvA1bGyicBfzWwnsMbdZ9L1a0nIa0yLREC05bPV3SfHF5pZJrAgtvsk0S/L+CbvSOCj2HYl8OfYF/9bZhYhujZIdSIDj9nv+N19U9zj7gKeTmTAHexv/BOAccCi2AdqJPCOmR3n7h8nOHbonf8/8f4IPEOSEgG9FH+sw/JM4JRk/ACK09vvf7J0GjeAu98H3AdgZq8Cl7j72rhDKoGT4vZHEu1LqCQRrzERnSZhuAFjieu0Ad4EvhzbNuCoLh73NtFf/e0dMWfEyr8J/Ci2fRDRZpulUPzD4o65Bngkld7/DsesJYGdxQl6/w+MO+ZK4LEUi38G8AFQksi4E/3/hwR2Fu9r3HTdWbyG6FmIwbHtIT15jfsUdzL+UZN9Ax4GNhK9AlolcBnRX5TPAYti/6F/0MVjy4AlwCrg1+yedJcDPBS77x3g8ykW/4PAe8Bior+ehqVS/B2OWUtiRw0l4v1/PFa+mOjaMCPmFz3pAAADVElEQVRSLP6VRH/8LIzdEjnqKRHxz4w9VzOwCXg+LHHTSSKIlV8ae99XAl/bm8/I3t40s1hEJM2l06ghERHphBKBiEiaUyIQEUlzSgQiImlOiUBEJM0pEUhKMrOGJNd3t5kd2kvP1WbRVUiXmNlT3a3kaWaDzOzbvVG3SGc0fFRSkpk1uHtBLz5flu9eVC2h4mM3sweA5e7+kz0cPxZ42t0PT0Z8kn7UIpA+w8xKzOxxM3s7dvtsrPw4M3vTzN6N/Z0UK7/EzB41s6eAF8zsJDN71cwes+ja+39oX+s9Vl4W226ILSC3yMzmm9kBsfIJsf23zexHPWy1zGP3wnoFZvaymb1j0fXmz44d81NgQqwVcXPs2Otj9Sw2s//fi2+jpCElAulLbgN+6e7HAucBd8fKlwHT3f1ooqt+/mfcY6YBF7v752P7RwPfAQ4FxgOf7aSefGC+ux8FzAUuj6v/tlj93a7/Elsr5xSiM70BmoCZ7n4M0etf3BJLRP8KrHL3ye5+vZmdDhwIHAdMBqaY2fTu6hPpSrosOifp4VTg0LiVHgeYWSEwEHjAzA4kulJjdtxjXnT3+DXk33L3SgAzW0h07ZjXO9Szk92L9i0ATottT2P32vB/BP6rizj7xT33AqJrzUN07Zj/jH2pR4i2FA7o5PGnx27vxvYLiCaGuV3UJ7JHSgTSl2QA09y9Mb7QzG4HXnH3mbHz7a/G3b29w3M0x2230flnpMV3d651dcyeNLr7ZDMbSDSh/BPwK6LXKSgBprh7i5mtBfI6ebwBN7n7nXtZr0indGpI+pIXiK7zD4CZtS//OxDYENu+JIH1zyd6Sgrg/O4OdvdtRC9beZ2ZZRONsyqWBE4GxsQOrQcK4x76PHBpbL17zGyEmZX20muQNKREIKmqv5lVxt2uJfqlWhbrQP2A6NLhAD8HbjKzN4DMBMb0HeBaM3sLGAZs6+4B7v4u0ZUpzyd6sZcyMysn2jpYFjumBngjNtz0Znd/geipp3lm9h7wGJ9MFCJ7RcNHRXpJ7Epqje7uZnY+cIG7n93d40SCpj4Ckd4zBfh1bKTPVpJ0KVCR/aUWgYhImlMfgYhImlMiEBFJc0oEIiJpTolARCTNKRGIiKQ5JQIRkTT3f4xAe0Gjeeh0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.callbacks import SaveModelCallback, ReduceLROnPlateauCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 22:51:40 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>3.655336</th>\n",
       "    <th>3.475370</th>\n",
       "    <th>0.390683</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>3.477082</th>\n",
       "    <th>3.311656</th>\n",
       "    <th>0.404194</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>3.415747</th>\n",
       "    <th>3.220839</th>\n",
       "    <th>0.413514</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>3.320024</th>\n",
       "    <th>3.138681</th>\n",
       "    <th>0.423168</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>5</th>\n",
       "    <th>3.305291</th>\n",
       "    <th>3.106956</th>\n",
       "    <th>0.427272</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(5, 2e-3, moms=(0.8, 0.7), callbacks=[SaveModelCallback(learn), ReduceLROnPlateauCallback(learn, factor=0.8)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(\"lm_5_ep_lr2-3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder(\"lm_5_ep_lr2-3_enc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEKCAYAAAARnO4WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHVpJREFUeJzt3XmUXGd55/HvU1Xd1d3qlmRZLbltWZIXLTbesNuAQ+IxBgy2GScEwzEHfFhmxiczA4RwwANDJgQIgZhkEkgmJA5gAsZwsI3nYAhehuCYeMGWZMuS900ttXa31K1eq2t55o97SyqLllQt1b23lt/nqE7Xcqvu86q6f/et9773lrk7IiLS/FJJFyAiIvFQ4IuItAgFvohIi1Dgi4i0CAW+iEiLUOCLiLQIBb6ISItQ4IuItAgFvohIi8gkXUClhQsX+vLly5MuQ0SkYaxdu/YVd++tZtm6Cvzly5ezZs2apMsQEWkYZjZQ7bIa0hERaREKfBGRFqHAFxFpEQp8EZEWocAXEWkRCnwRkRahwBcRaREKfBGRBN371E7+4d9ejGVdCnwRkQT9fON2vvvgpljWpcAXEUnQjpEp+uZ3xrIuBb6ISIK2j0xxwryOWNalwBcRSYi7s31kkhMV+CIizW14Is9UvsQJ8zSkIyLS1LaPTAGohy8i0ux27JsE0Bi+iEiz2zYc9PD7NKQjItLcdoxMkU4ZvT3ZWNanwBcRSci2kUkW92RJpyyW9SnwRUQSEudBV6DAFxFJTJwHXYECX0QkEXEfdAUKfBGRRMR90BUo8EVEElE+6KpPPXwRkea2fSQ46KopAt/MVpnZ4xWXfWb28ajWJyLSSA708OMb0slE9cLu/ixwHoCZpYGtwB1RrU9EpJFsH5mM9aAriG9I583Ai+4+ENP6RETq2vaRqVgPuoL4Av8a4AcxrUtEpO7tiHkOPsQQ+GbWDlwF3HqIx68zszVmtmb37t1RlyMiUhe2x3yULcTTw78cWOfuO2d60N1vdPd+d+/v7e2NoRwRkWSVD7rqm9tkPXzgvWg4R0Rkv/JBV03VwzezLuCtwI+jXI+ISCNJ4qAriHBaJoC7TwDHR7kOEZFGk8RBV6AjbUVEYpfEQVegwBcRiV0SB12BAl9EJHZJHHQFCnwRkdhtH47/oCtQ4IuIxG7HvvgPugIFvohIrNydbcPxH3QFCnwRkVgNT+TJFeI/6AoU+CIisdqW0Bx8UOCLiMRqRzgHXzttRUSaXPmgqxNjPugKFPgiIrFK6qArUOCLiMQqqYOuQIEvIhKrpA66AgW+iEistg5PJjIlExT4IiKxyRdLbB2eZNmCrkTWr8AXEYnJtuFJiiVn2fEKfBGRpjYwNAHAsuPnJLJ+Bb6ISEwG9pQDXz18EZGmtnlonPZMisU9mqUjItLUBoYmWLqgi1QCc/BBgS8iEpvNeyYSm6EDCnwRkVi4e9DDT2j8HhT4IiKx2D2aYzJfVA9fRKTZ7Z+hszCZKZmgwBcRicX+Ofjq4YuINLfNQ+OkDJYcp8AXEWlqA3sm6JvXSXsmudhV4IuIxGBgaCKxI2zLFPgiIjHYvEeBLyLS9PZN5dkzPs3SBcnN0AEFvohI5DaHM3SWq4cvItLcylMykzzKFhT4IiKRG9gzDiR3HvwyBb6ISMQ2D01w/Jx2urOZROtQ4IuIRCzpk6aVKfBFRCKW9GmRyxT4IiIRyhWKbBuZZGnC4/egwBcRidTg3knck5+SCQp8EZFIDQyVZ+g0eeCb2Xwzu83MnjGzp83soijXJyJSb/bPwU/4KFuAqOcIfQ24y92vNrN2IPlNnIhIjAaGJuhqT7Owuz3pUqILfDObC1wMfBDA3aeB6ajWJyJSjzbvmWDpgi7MLOlSIh3SORXYDdxkZo+Z2TfNLPnPNCIiMRoYGq+L8XuINvAzwPnAN9z9tcA48OmDFzKz68xsjZmt2b17d4TliIjEq1RytuydTPyUCmVRBv4gMOjuvw5v30awAXgVd7/R3fvdvb+3tzfCckRE4rV1eJLpQolTEvzi8kqRBb677wC2mNmq8K43A09FtT4RkXrz3M5RAFYu7k64kkDUs3Q+Cnw/nKHzEvChiNcnIlI3nts5BsDpi3oSriQQaeC7++NAf5TrEBGpV8/vGmXx3CzzOtuSLgXQkbYiIpF5fucYKxfXR+8eFPgiIpEolZwXdo2xok6Gc0CBLyISicG9k0zmi3WzwxYU+CIikXh+VzBDZ4UCX0SkudXbDB1Q4IuIROL5naOcMLejbmbogAJfRCQSz+0aravhHFDgi4jUXD3O0AEFvohIzQ3unWQqX6qrGTqgwBcRqbnyOXRW1NFBV6DAFxGpuefqcEomKPBFRGruhZ1j9M3rYG5H/czQAQW+iEjNPbdrlNMX1VfvHhT4IiI1VZ6hU08nTStT4IuI1NCWvRN1OUMHFPgiIjX1fHhKhXqboQMKfBGRmto/Q6dRx/DN7DQzy4bXLzGzj5nZ/GhLExFpPM+HM3R66myGDlTfw78dKJrZ6cC3gFOAWyKrSkSkQT23c7Quh3Og+sAvuXsBeCfwN+7+R0BfdGWJiDSeYnmGTh0O50D1gZ83s/cCHwB+Gt5Xf59XREQSNLh3glyhVJdTMqH6wP8QcBHwJXd/2cxOAW6OriwRkcbzzI76PKVCWaaahdz9KeBjAGZ2HNDj7l+JsjARkUazcesI6ZRxRt/cpEuZUbWzdO4zs7lmtgBYD9xkZv872tJERBrLE4MjrFjUTUdbOulSZlTtkM48d98H/D5wk7tfALwlurJERBqLu7Nx6wjnLJmXdCmHVG3gZ8ysD3gPB3baiohIaNvIFEPj05x9UuMH/heAu4EX3f1RMzsVeD66skREGsuGwREAzl5Sv8ekVrvT9lbg1orbLwHviqooEZFGs2HrMJmUsfqE+pySCdXvtF1iZneY2S4z22lmt5vZkqiLExFpFBu27mPl4p663WEL1Q/p3AT8BDgROAm4M7xPRKTluTsbBofrevweqg/8Xne/yd0L4eU7QG+EdYmINIytw5Psnchzdh3P0IHqA/8VM3u/maXDy/uBoSgLExFpFPt32DZJD//DBFMydwDbgasJTrcgItLyNmwdoS1trO6r3x22UGXgu/tmd7/K3XvdfZG7/x7BQVgiIi1vw9YRVi7uIZup3x22cGzfePWJmlUhItKg3J0NdX6EbdmxBL7VrAoRkQY1uHeS4Yk8Z9X5+D0cW+B7zaoQEWlQG7YGO2zPOal+j7AtO+yRtmY2yszBbkBnJBWJiDSQJwaDHbYrT6jPc+BXOmzgu3t973IWEUnYhq3DrD5hbt3vsIUqz6VztMxsEzAKFIGCu/dHuT4RkTgFR9iOcOU5JyZdSlUiDfzQm9z9lRjWIyISq817Jtg3VWiIGTpwbDttRURaWnmHbb0fYVsWdeA7cI+ZrTWz6yJel4hIrNZvGaY9k2Ll4sbY3Rn1kM4b3X2bmS0C7jWzZ9z9/soFwg3BdQBLly6NuBwRkdpZM7CXc5fMoz3TGIMlkVbp7tvCn7uAO4DXzbDMje7e7+79vb06AaeINIbJ6SIbt47Qv3xB0qVULbLAN7M5ZtZTvg5cBmyMan0iInFaPzhMvuhcuPy4pEupWpRDOouBO8ysvJ5b3P2uCNcnIhKbNZv2AHDB0sbp4UcW+OH33p4b1euLiCTp0U17WbW4h3ldbUmXUrXG2NMgIlJHiiVn3cBeLmig4RxQ4IuIzNqzO0YZzRUaavweFPgiIrO2diAYv+9f1jjj96DAFxGZtUc37eWEuR0sOa6xThqswBcRmaU1m/bQv/w4wlmIDUOBLyIyC1uHJ9k2MsWFDXTAVZkCX0RkFvbPv1/WWDtsQYEvIjIrj27aQ3c2w+oTGuOEaZUU+CIis7Bm015eu3Q+mXTjxWfjVSwikpCRyTzP7hxtyPF7UOCLiFRt3ea9uEN/gx1wVabAFxGp0ppNe8ikjPNOnp90KUdFgS8iUqUHXxziNSfNo6s9jq8Drz0FvohIFbaPTPLY5mEuO3Nx0qUcNQW+iEgV7tq4A4DLzzoh4UqOngJfRKQK/7JhO6tP6OHU3u6kSzlqCnwRkSPYuW+KNQN7ueLsvqRLOSYKfBGRI7j7yR24wxVnN+5wDijwRUSO6F82bGfFom5OX9R4p1OopMAXETmM3aM5Hnl5D5c3+HAOKPBFRA7r7id3UGqC4RxQ4IuIHNbPN27n1N45rFrc2MM5oMAXETmkobEcD7+0hyvO6mu4b7eaiQJfROQQ7n1qJ8WSc3kTDOeAAl9E5JB+tmE7y47v4sy+uUmXUhMKfBGRGewYmeLBF4e48uzmGM4BBb6IyIxueWQzJXeuuXBp0qXUjAJfROQg+WKJHzyymUtW9rL0+K6ky6kZBb6IyEHufnIHu0dzXHvRsqRLqSkFvojIQb730AAnL+jkP6xclHQpNaXAFxGp8OyOUX798h7e9/plpFPNsbO2TIEvIlLh5ocHaM+keE//yUmXUnMKfBGR0FiuwI/XDfKOc/pYMKc96XJqToEvIhK6Y90g49NFrn1Dc+2sLVPgi4gA7s73Hh7g7JPmcd7J85MuJxIKfBER4BdP7+K5nWNc+4ZlTXNk7cEU+CLS8nKFIn/2s6c4rXcO7zz/pKTLiYwCX0Ra3nce2MSmoQn+1zvOpC3dvLHYvC0TEanCrtEp/vZfX+DS1Yu4ZFVzHWh1sMgD38zSZvaYmf006nWJiMzWV+96llyhyB9feUbSpUQujh7+HwJPx7AeEZFZWb9lmFvXDvKhN57Cqb3dSZcTuUgD38yWAFcC34xyPSIis+XufP7OJ1nY3c5HLz096XJiEXUP/2+A64FSxOsREZmV29dtZd3mYa5/22p6OtqSLicWkQW+mb0D2OXua4+w3HVmtsbM1uzevTuqckRE9tsxMsUX7nySC5Ydx9UXLEm6nNhE2cN/I3CVmW0CfghcamY3H7yQu9/o7v3u3t/b2xthOSIiwVDOZ378BNPFEn/57nNJNdkZMQ8nssB398+4+xJ3Xw5cA/yru78/qvWJiFTj1jWD/PLZ3fyPt6/mlIVzki4nVpqHLyItY+vwJF/86VO8/pQFfOCi5UmXE7tMHCtx9/uA++JYl4jITNydT9/+BEV3vnp1aw3llKmHLyIt4ZZHNvOr51/hf15xRlN9MflsKPBFpOltHprgSz97mt8+fSHve/3SpMtJjAJfRJpaqeR88tb1pM244epzmvbUx9VQ4ItIU/v2Ay/zyKY9fO6q13Di/M6ky0mUAl9EmtYLu0a54e5necsZi3lXE5/nvloKfBFpSvliiU/8aD1z2tN8+ffPbumhnLJYpmWKiMTtG/e9yBODI/z9+86ntyebdDl1QT18EWk6u0an+D+/fIErz+njirP7ki6nbijwRaTp/NP9L5EvlvjUZauSLqWuKPBFpKkMjeW4+eHNXHXuiSxvsXPlHIkCX0Sayrf+/WWmCkU+0iJfajIbCnwRaRrDE9N896EBrjirj9MX9SRdTt1R4ItI07jpgU2M5Qrq3R+CAl9EmsLoVJ6bHniZt565mDP65iZdTl1S4ItIU/juQwPsmyrwsUtXJF1K3VLgi0jD27Vvim/+6iUuWdXL2UvmJV1O3VLgi0hDG56Y5tpvPUKuUOL6t61Oupy6psAXkYY1nivwoe88ysuvjHPjtf2ceaLG7g9HgS8iDSlXKPIHN69l/ZZhvv7e8/jtFQuTLqnu6eRpItJwiiXn4z98nF89/wo3XH0Obz9L58uphnr4ItJQSiXn+tue4Ocbd/DHV57Be/pPTrqkhqHAF5GG4e587idPcvu6Qf7wzSv4z79zatIlNRQFvog0BHfnKz9/hu89PMB1F5/Kx9+i+fazpcAXkYbw9V+8wD/e/xLXvmEZn7l8tb7B6ihop62I1K1iyXnwxVf40ZpB7ly/jXedv4TPX/Uahf1RaorAv3XNFrqzGRb2ZOntztLbk6WjLU2uUCSXL5ErlJgulMiXShSKTr5Ywh0621PMyWaYk83Q1ZamUHImpotMTBeYnC5iBtlMmo62NJ3taTrb0qRT+kUTidK+qTzP7Rjl3qd38n8f28rOfTnmdmS47uJTuf5tq0jpb/CoNXzguzufvWMj08VSLOvryWaY29nG3M42ejoydLal6WhL0dGWpiOTZk42Q3dHhu5smu5sG4t6sixZ0MmS47rozh7df3e+WGI8V2B8ush4rkDJnbZ0ivZ0irZ0inTKcJzwHwZ0tqfpas/MuIEqlfxVG7980QGC18sYmVSKTMpwoOROyR334PFj/WMrlpzhiWn2jE+zdyJPOkWwQa3YqHa2p2lPp9SLayLFkjM+XWBsqsB4rsC+qQJDYzmGxqcZGsuxezTHS6+M8/zOMXbsmwIgkzIuWbWIz/3Hk7h09SI62tIJt6LxNXzgAzzw6UvZPZrjlfAXZ/dYjly+REdbimwmRbYtCJBM2mhLB2FmZkzmgwANLkUyaaOrPU1Xe5rO9uC/Zipf3H8ZyxXZN5ln31Q+/FlgeGKaqXyJXKHIZL7IRK7I2HQB99+s87iuNuZ3tVMolSiVoFAKPmlkUkYmrC+TMnKFEpPT5fWWjmljls2k6GxPByEfBnyhNENxVWpPH/g/bUtbWHNqfxvaMyna00Z7JoVhTEwXmJguMj4d/B8PT0xTzerTKaOrLU1He5pspvy6wXo721LhhjbYQHRl08xpz9DVnmFONr3/Z3f46S2bSVHyoHPggDukU5CyoPZUiqAN4f9/OrykrHwBM3vV80t+YGNZKAafInOFIpPTJSbD35fJit+dyekSRXcMMAPDSKcgnUod+BmuJ3g8uD4nm6Yn2xZ2IjLMCzsaPR1tR/1pc7oQdCDGwku5zly+xFS+yHQx6AwU3SmWnELJyRdK5IvBJ+XpYrDcxHRlG4PHcoUiuULF49MHljuc7myG5Qu7+K3TjmfF4h5WLu7mvJPnc3y3vny8lho+8M2M3p5sXX0rfankTOSLjE7l2bkvx5Y9EwzunWTL3gn2Teb398rT4R93oRSERr7oFEql/Z8WOtvTZNtSzGkPgqs7G3yCSJsxHS6fL5YoFEtgtj9M3IMN1XiuuD9w0ymjLdzgtYXBnEmVbwfBkd/f4w82CpVhZ0b4B13a/wdeCJcrlJxiqcR0wfeHQi5fouTOnGyG3p4sXe0ZutrTLJjT/qqLO/tDY6IcDmHNE+FGb7pQIlcshcNzQTANjU8zGQbJZLhBmcrH8ymvXnRnM3RUbHjbUikw9gf3VD4I30pO0Ns+Vp1tQceoPNwZdK6CjXN3NkM2U+44HehAzQ03WuWN1/FzshzfHfweqPcej4YP/HqUSlnwi53N0Devk/NOnp90SS2hUCwxHu6DCXqwwSe46UKJVCrYIKbCYaKiO6XSgR5s8LMU/CwGw1hFd0oebMDLKnvf7ZnUqzailcNS5WG+8ieRjnD/j4fDY+XgLfmB9RdL/hufIiamg47D2FQwDDKWK1R8yiwwVSgGG96ikw9fryNzYP3ZTIrKkTH3IKy7O8qdiEwQ2BXPKXcG0hWX9rCTUPkJWRqPAl+aRiadYl5ninmdbUmXckjlT0uAJgBI7DQPX0SkRSjwRURahAJfRKRFKPBFRFqEAl9EpEUo8EVEWoQCX0SkRSjwRURahPlMJ31JiJntBgYq7poHjBy02JHuO9L1hcArx1DmTOufzXLVtOlwt5Nq0+GWUZt+83b5euV9atPs6q1mmdm2qZrrx9KmKPJhpvsrb69w93lVVRcc6l2fF+DG2d53pOvAmlrXNJvlqmnT4W4n1abDLaM2HbYdlfepTQm3qcrrR92mKPJhNm060qXeh3TuPIr7qrl+LKp9nUMtV02bDnc7qTYdbhm16Tdv33mIZY6F2lTdY0fzu1bP+TDT/UdVe10N6cTBzNa4e3/SddSS2tQY1KbG0IxtKqv3Hn4Ubky6gAioTY1BbWoMzdgmoAV7+CIiraoVe/giIi2poQPfzL5tZrvMbONRPPcCM9tgZi+Y2det4hsdzOyjZvasmT1pZjfUtuoj1lXzNpnZn5rZVjN7PLxcUfvKD1tXJO9T+PgnzczNbGHtKq6qrijepy+a2RPhe3SPmZ1Y+8oPWVMU7fmqmT0TtukOM4v1m4AiatO7w1womVnjjfMfy5SqpC/AxcD5wMajeO4jwEUEX2D0c+Dy8P43Af8PyIa3FzVBm/4U+GQzvU/hYycDdxMcu7Gw0dsEzK1Y5mPAPzR4ey4DMuH1vwD+ogneozOAVcB9QH+c7anFpaF7+O5+P7Cn8j4zO83M7jKztWb2KzNbffDzzKyP4I/rIQ/exe8Cvxc+/F+Br7h7LlzHrmhb8WoRtSlREbbpr4HrCb4xMFZRtMnd91UsOocY2xVRe+5x90K46MPAkmhb8WoRtelpd382jvqj0NCBfwg3Ah919wuATwJ/P8MyJwGDFbcHw/sAVgK/Y2a/NrN/M7MLI622OsfaJoCPhB+tv21mx0VXatWOqU1mdhWw1d3XR13oLBzz+2RmXzKzLcD7gD+JsNZq1OL3ruzDBD3lpNWyTQ2nqb7T1sy6gd8Cbq0Y6s3OtOgM95V7UxngOOANwIXAj8zs1HBLH7satekbwBfD218E/orgDzARx9omM+sCPkswZFAXavQ+4e6fBT5rZp8BPgJ8rsalVqVW7Qlf67NAAfh+LWucrVq2qVE1VeATfGIZdvfzKu80szSwNrz5E4IArPx4uQTYFl4fBH4cBvwjZlYiOLfG7igLP4xjbpO776x43j8BP42y4Coca5tOA04B1od/uEuAdWb2OnffEXHth1KL371KtwA/I6HAp0btMbMPAO8A3pxUp6lCrd+jxpP0ToRjvQDLqdgpAzwIvDu8bsC5h3jeowS9+PJOmSvC+/8A+EJ4fSWwhfB4hQZuU1/FMn8E/LDR36eDltlEzDttI3qfVlQs81HgtgZvz9uBp4DeuN+bqH/vaNCdtokXcIxv5g+A7UCeoGf+nwh6fncB68Nftj85xHP7gY3Ai8DflUMdaAduDh9bB1zaBG36HrABeIKgB9MXV3uiatNBy8Qe+BG9T7eH9z9BcH6Ukxq8PS8QdJgeDy+xzTqKsE3vDF8rB+wE7o6zTcd60ZG2IiItohln6YiIyAwU+CIiLUKBLyLSIhT4IiItQoEvItIiFPhS18xsLOb1fdPMzqzRaxXDM19uNLM7j3S2SDObb2b/rRbrFpmJpmVKXTOzMXfvruHrZfzACb0iVVm7mf0z8Jy7f+kwyy8HfuruZ8VRn7Qe9fCl4ZhZr5ndbmaPhpc3hve/zsweNLPHwp+rwvs/aGa3mtmdwD1mdomZ3Wdmt4Xna/9+xfnO7yuf59zMxsKTma03s4fNbHF4/2nh7UfN7AtVfgp5iAMnfus2s1+Y2ToLzrn+u+EyXwFOCz8VfDVc9lPhep4ws8/X8L9RWpACXxrR14C/dvcLgXcB3wzvfwa42N1fS3CmyT+veM5FwAfc/dLw9muBjwNnAqcCb5xhPXOAh939XOB+4L9UrP9r4fqPeI6V8FwtbyY4yhlgCninu59P8P0LfxVucD4NvOju57n7p8zsMmAF8DrgPOACM7v4SOsTOZRmO3matIa3AGdWnPFwrpn1APOAfzazFQRnN2yreM697l55bvRH3H0QwMweJzjnyr8ftJ5pDpxobi3w1vD6RRw4L/8twF8eos7OitdeC9wb3m/An4fhXSLo+S+e4fmXhZfHwtvdBBuA+w+xPpHDUuBLI0oBF7n7ZOWdZva3wC/d/Z3hePh9FQ+PH/QauYrrRWb+W8j7gZ1ch1rmcCbd/Twzm0ew4fjvwNcJznXfC1zg7nkz2wR0zPB8A77s7v84y/WKzEhDOtKI7iE4VzwAZlY+3e08YGt4/YMRrv9hgqEkgGuOtLC7jxB8ZeEnzayNoM5dYdi/CVgWLjoK9FQ89W7gw+F53DGzk8xsUY3aIC1IgS/1rsvMBisunyAIz/5wR+ZTBKe0BrgB+LKZPQCkI6zp48AnzOwRoA8YOdIT3P0xgjM0XkPwRSD9ZraGoLf/TLjMEPBAOI3zq+5+D8GQ0UNmtgG4jVdvEERmRdMyRWYp/MatSXd3M7sGeK+7/+6RnieSNI3hi8zeBcDfhTNrhknw6yJFZkM9fBGRFqExfBGRFqHAFxFpEQp8EZEWocAXEWkRCnwRkRahwBcRaRH/H3Q+8tAn5vLJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 22:53:02 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>3.327268</th>\n",
       "    <th>3.126424</th>\n",
       "    <th>0.424546</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>3.309945</th>\n",
       "    <th>3.125146</th>\n",
       "    <th>0.424225</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>3.305685</th>\n",
       "    <th>3.107883</th>\n",
       "    <th>0.426271</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>3.311324</th>\n",
       "    <th>3.091680</th>\n",
       "    <th>0.428564</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>5</th>\n",
       "    <th>3.258298</th>\n",
       "    <th>3.086017</th>\n",
       "    <th>0.429533</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(5, slice(2e-3 / 20, 1e-3), moms=(0.8, 0.7), callbacks=[SaveModelCallback(learn, name=\"best_model_2\")])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(\"lm_5_ep_lr2-3_5_stlr\")\n",
    "learn.save_encoder(\"lm_5_ep_lr2-3_5_stlr_enc\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 477,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Россиян шокировали обладателями бюджетных средств . \\n\\n\\n \" xxmaj граждане не ответили и не хотели бы xxup'"
      ]
     },
     "execution_count": 477,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.predict(\"Россиян шокировали\", 15)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}