training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Skin Cancer Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Put these at the top of every notebook, to get automatic reloading and inline plotting\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from fastai.vision import *\n",
    "from fastai.metrics import error_rate, accuracy\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "from transform_to_pb_file import export_to_pb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training: nb melanoma images: 1174\n",
      "Training: nb other images: 1626\n",
      "Valid: nb melanoma images: 30\n",
      "Valid: nb other images: 120\n",
      "Test: nb melanoma images: 117\n",
      "Test: nb other images: 483\n"
     ]
    }
   ],
   "source": [
    "print(\"Training: nb melanoma images:\", len(os.listdir('data/train/melanoma')))\n",
    "print(\"Training: nb other images:\", len(os.listdir('data/train/other')))\n",
    "print(\"Valid: nb melanoma images:\", len(os.listdir('data/valid/melanoma')))\n",
    "print(\"Valid: nb other images:\", len(os.listdir('data/valid/other')))\n",
    "print(\"Test: nb melanoma images:\", len(os.listdir('data/test/melanoma')))\n",
    "print(\"Test: nb other images:\", len(os.listdir('data/test/other')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"data/\"\n",
    "sz = 200\n",
    "bs = 64\n",
    "classes = ('melanoma', 'other')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "data = ImageDataBunch.from_folder(path, train=\"train\", valid=\"valid\", test=\"test\", \n",
    "        ds_tfms=get_transforms(flip_vert=True), size=sz, num_workers=4).normalize(imagenet_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['melanoma', 'other']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['melanoma', 'other'], 2, 2800, 150, 600)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.classes, data.c, len(data.train_ds), len(data.valid_ds), len(data.test_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VGG16Like(nn.Module):\n",
    "    def __init__(self, shrink=True, dense_units=None):\n",
    "        assert shrink or dense_units is not None\n",
    "        super().__init__()\n",
    "        \n",
    "        # Input size: batch x img_depth x 200 x 200\n",
    "        self.block1 = self.two_conv_block(in_channel=3, out_channel=64,\n",
    "            kernel=3, stride=1, padding=1, pool_kernel=2)\n",
    "        # Input size: batch x 64 x 100 x 100\n",
    "        self.block2 = self.two_conv_block(64, 128, 3, 1, 1, 2)\n",
    "        # Input size: batch x 128 x 50 x 50\n",
    "        self.block3 = self.three_conv_block(128, 256, 3, 1, 1, 2)\n",
    "        # Input size: batch x 256 x 25 x 25\n",
    "        self.block4 = self.three_conv_block(256, 512, 3, 1, 1, 2)\n",
    "        # Input size: batch x 512 x 6 x 6\n",
    "        self.block5 = self.three_conv_block(512, 512, 3, 1, 1, 2)\n",
    "        self.dense = self.dense_block(\n",
    "            in_channel=512 * 6 * 6,\n",
    "            out_channel=dense_units,\n",
    "            nb_classes=2,\n",
    "            shrink=shrink,\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.block1(x)\n",
    "        x = self.block2(x)\n",
    "        x = self.block3(x)\n",
    "        x = self.block4(x)\n",
    "        x = self.block5(x)\n",
    "        x = x.view(-1,512 * 6 * 6)\n",
    "        x = self.dense(x)\n",
    "        return F.log_softmax(x, dim=-1)\n",
    "\n",
    "    @classmethod\n",
    "    def two_conv_block(cls, in_channel, out_channel, kernel, stride, padding, pool_kernel):\n",
    "        return nn.Sequential(\n",
    "            nn.Conv2d(in_channel, out_channel, kernel, stride=stride, padding=padding),\n",
    "            nn.BatchNorm2d(out_channel),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(out_channel, out_channel, kernel, stride=stride, padding=padding),\n",
    "            nn.BatchNorm2d(out_channel),\n",
    "            nn.ReLU(),\n",
    "            nn.AvgPool2d(pool_kernel),\n",
    "        )\n",
    "    \n",
    "    @classmethod\n",
    "    def three_conv_block(cls, in_channel, out_channel, kernel, stride, padding, pool_kernel):\n",
    "        return nn.Sequential(\n",
    "            nn.Conv2d(in_channel, out_channel, kernel, stride=stride, padding=padding),\n",
    "            nn.BatchNorm2d(out_channel),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(out_channel, out_channel, kernel, stride=stride, padding=padding),\n",
    "            nn.BatchNorm2d(out_channel),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(out_channel, out_channel, kernel, stride=stride, padding=padding),\n",
    "            nn.BatchNorm2d(out_channel),\n",
    "            nn.ReLU(),\n",
    "            nn.AvgPool2d(pool_kernel),\n",
    "        )\n",
    "    \n",
    "    @classmethod\n",
    "    def dense_block(cls, in_channel, out_channel, nb_classes, shrink=True):\n",
    "        if shrink:\n",
    "            # Chop off the original dense layers\n",
    "            return nn.Linear(in_channel, nb_classes)\n",
    "        return nn.Sequential(\n",
    "            nn.Linear(in_channel, out_channel),\n",
    "            nn.Linear(out_channel, out_channel),\n",
    "            nn.Linear(out_channel, out_channel),\n",
    "            nn.Linear(out_channel, nb_classes),\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = VGG16Like()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(data, model, loss_func = nn.CrossEntropyLoss(), metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======================================================================\n",
      "Layer (type)         Output Shape         Param #    Trainable \n",
      "======================================================================\n",
      "Conv2d               [64, 64, 200, 200]   1792       True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 64, 200, 200]   128        True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 64, 200, 200]   0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 64, 200, 200]   36928      True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 64, 200, 200]   128        True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 64, 200, 200]   0          False     \n",
      "______________________________________________________________________\n",
      "AvgPool2d            [64, 64, 100, 100]   0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 128, 100, 100]  73856      True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 128, 100, 100]  256        True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 128, 100, 100]  0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 128, 100, 100]  147584     True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 128, 100, 100]  256        True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 128, 100, 100]  0          False     \n",
      "______________________________________________________________________\n",
      "AvgPool2d            [64, 128, 50, 50]    0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 256, 50, 50]    295168     True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 256, 50, 50]    512        True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 256, 50, 50]    0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 256, 50, 50]    590080     True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 256, 50, 50]    512        True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 256, 50, 50]    0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 256, 50, 50]    590080     True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 256, 50, 50]    512        True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 256, 50, 50]    0          False     \n",
      "______________________________________________________________________\n",
      "AvgPool2d            [64, 256, 25, 25]    0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 512, 25, 25]    1180160    True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 512, 25, 25]    1024       True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 512, 25, 25]    0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 512, 25, 25]    2359808    True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 512, 25, 25]    1024       True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 512, 25, 25]    0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 512, 25, 25]    2359808    True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 512, 25, 25]    1024       True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 512, 25, 25]    0          False     \n",
      "______________________________________________________________________\n",
      "AvgPool2d            [64, 512, 12, 12]    0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 512, 12, 12]    2359808    True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 512, 12, 12]    1024       True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 512, 12, 12]    0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 512, 12, 12]    2359808    True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 512, 12, 12]    1024       True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 512, 12, 12]    0          False     \n",
      "______________________________________________________________________\n",
      "Conv2d               [64, 512, 12, 12]    2359808    True      \n",
      "______________________________________________________________________\n",
      "BatchNorm2d          [64, 512, 12, 12]    1024       True      \n",
      "______________________________________________________________________\n",
      "ReLU                 [64, 512, 12, 12]    0          False     \n",
      "______________________________________________________________________\n",
      "AvgPool2d            [64, 512, 6, 6]      0          False     \n",
      "______________________________________________________________________\n",
      "Linear               [64, 2]              36866      True      \n",
      "______________________________________________________________________\n",
      "\n",
      "Total params: 14760002\n",
      "Total trainable params: 14760002\n",
      "Total non-trainable params: 0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(learn.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xt8XHWd//HXJ/d7mjbp/UophXKnoYIIoqxa+aGAF6TKyh33AjzUH/rTn7vgwo8VV11xFZctLHfFBdQVXBBWVkCBliYCpS20lLZp09I2TdI092RmPr8/ZhqGkKRpMydnJnk/H495dM5l5ny+nXbec873nO8xd0dERAQgK+wCREQkfSgURESkj0JBRET6KBRERKSPQkFERPooFEREpI9CQURE+igURESkj0JBRET65IRdwMGqrKz0uXPnhl2GiEhGqa2t3ePuVQdaL+NCYe7cudTU1IRdhohIRjGzuuGsp8NHIiLSR6EgIiJ9FAoiItJHoSAiIn0UCiIi0kehICIifRQKIiLSR6EgIpLmYjHnx0+/ydodLYFvK+MuXhMRGU/auyP874de5Xdrd9LRG+Xo6eWBbk+hICKSprY1dXDlfTVs2NXK3/2vo7j8A/MC36ZCQUQkDb20uYm/eqCW3miMuy9dwgePOOCwRSmhUBARSTOPv/Y21z74MrMnFnHHxdXMryoZtW0rFERE0kjNlia+/B+vcPysCdx1ycmUF+aO6vZ19pGISJrYvKedK+6rYcaEQu78YvWoBwIoFERE0kJjWzeX3P0SWWbcc+nJVBTnhVKHQkFEZBTFYs6GXa00tnXj7gB09Ua58r4adrZ0cccXq5kzqTi0+tSnICIyiu5+YQs3/XYdAAW5WUyfUIgBm/a089PPn8TiORWh1qdQEBEZJZFojLv+tJnjZ5Zz3okz2LG3kx17u9i5r4tbPnUYHz92WtglBhcKZnYXcA6w292PGWD5ucBNQAyIAF929z8FVY+ISNieWreL7Xs7ueETi/jo0VPDLmdAQfYp3AMsHWL508Dx7n4CcBlwZ4C1iIiE7t//tJnZE4s466gpYZcyqMBCwd2fA5qGWN7m+3tZoBjwwdYVEcl0r2zbS21dM5eeNpfsLAu7nEGFevaRmZ1vZm8A/0V8b0FEZEy660+bKc3P4bPVs8IuZUihhoK7/9rdjwTOI96/MCAzu8rMasyspqGhYfQKFBFJgbdbOnn8tbf53MmzKMlP7/N70uI6hcShpvlmVjnI8uXuXu3u1VVVozMolIhIqtz3Yh0xdy5+/9ywSzmg0ELBzA43M0s8PwnIAxrDqkdEJAgdPRF+vnIrHzt6KrMmFoVdzgEFeUrqg8CZQKWZ1QM3ALkA7n478Gngi2bWC3QCn0vqeBYRGRN+9efttHT2jsq9EFIhsFBw92UHWP5d4LtBbV9EJB08+NJWjptZHvqVysOVFn0KIiJjVV1jB4vnVJA4Wp72FAoiIgHpjkRp644wKaQRTw+FQkFEJCBN7T0ATCzOD7mS4VMoiIgEpLFtfyhoT0FEZNxrTOwpVJYoFERExr2m9m5AewoiIsI7h48mqU9BRESa2nvIyTLKCtN7vKNkCgURkYA0tfdQUZyXMdcogEJBRCQwe9p6MuoaBVAoiIgEpqm9m0kZdOYRKBRERALT1N6TUReugUJBRCQwje06fCQiIkBPJEZrVySjrlEAhYKISCCaOxLXKKhPQURE9rTFr2bW4SMREcnIEVJBoSAiEoh3QkF7CgCY2V1mttvM1gyy/AtmtjrxeMHMjg+qFhGR0fbOuEcKhf3uAZYOsXwz8EF3Pw64CVgeYC0iIqOqqb2H7CyjvDA37FIOSmCjNLn7c2Y2d4jlLyRNrgBmBlWLiMhoa2zvpqIoj6yszBn3CNKnT+Fy4ImwixARSZXGDBz3CALcUxguM/sQ8VD4wBDrXAVcBTB79uxRqkxE5NDFh7jIvFAIdU/BzI4D7gTOdffGwdZz9+XuXu3u1VVVVaNXoIjIIWpq72Fihl24BiGGgpnNBn4F/KW7bwirDhGRIDS291CZgXsKgR0+MrMHgTOBSjOrB24AcgHc/XbgemAS8NPEDSgi7l4dVD0iIqOlNxqjpbM34y5cg2DPPlp2gOVXAFcEtX0RkbA0779wTYePRESksT0zL1wDhYKISMpl6hAXoFAQEUm5/XsKlTp8JCIijYlhszOxo1mhICKSYk3tPWQZTMiwcY9AoSAiknKN7T0ZOe4RKBRERFKuqS0zh7gAhYKISMo1tfdk3L2Z91MoiIik2J72biZlYCczKBRERFIuU0dIBYWCiEhKRaIx9nb0KhRERASaO3oB1KcgIiLvDHGhPgUREUm6mll7CiIi417fCKk6fCQiIpk8QiooFEREUqqxvQczqChSKIiIjHtN7d1UFOWRnYHjHoFCQUQkpTL5wjUIMBTM7C4z221mawZZfqSZvWhm3WZ2XVB1iIiMpj0ZPBgeBLuncA+wdIjlTcC1wPcDrEFEZFQ1tfdk5L2Z9wssFNz9OeJf/IMt3+3uq4DeoGoQERltOnwkIiIARGNOc0cPk0oy82pmyJBQMLOrzKzGzGoaGhrCLkdEZEB7O3pwR4ePgubuy9292t2rq6qqwi5HRGRAjRl+4RpkSCiIiGSCxrb9g+FlbijkBPXGZvYgcCZQaWb1wA1ALoC7325mU4EaoAyImdmXgUXuvi+omkREghKJxnj8tbcBqCzN3D6FwELB3ZcdYPlOYGZQ2xcRGS07W7q49sGXeWlLE8uWzGLB5JKwSzpkgYWCiMh48Mz63Xz1oVfp6o3yw88dz/knZvZvXYWCiMghuvX3G7j1929y5NRSfvL5kzg8g/cQ9lMoiIgcgjue28Stv3+TT500g388/1gKcrPDLiklFAoiIgfpN69s5+bHX+fsY6fyvc8cn7Ejog5EoSAi0k9tXRM3/9frOHDtWQs484gqzOJf/M9v3MN1D7/K++ZN5J8vOGFMBQIoFERE+uxp6+aWJ97gkdp6ppUXkJ1lXHr3KqrnVHDdxxZSWpDDl+6vZX5VCcu/WD1mDhklUyiIyLjn7jywcivf+90bdPZG+esz53P1hw4nNzuLh2q28eP/eZMLl68gPyeLScV53HPpEsoLc8MuOxAKBREZ9x5/bSd//59rOO3wSfzDJ49511lEF50yh88snskDK+p4at0ubj7vGKaWF4RYbbAUCiIy7t31/GbmTCri/sveR9YAfQQFudlccfphXHH6YSFUN7o09pGIjGur6/dSW9fMxafOHTAQxhuFgoiMa/e8sIXivGw+U53ZVyKnikJBRMathtZufvvq23xm8UzKCsZmx/HBUiiIyLj185Vb6YnG+OL754ZdStpQKIjIuNQTifHAyjrOXFjF/KrMH7MoVRQKIjIuPbHmbRpau7lEewnvolAQkXHp7ue3cFhlMWcs0C1+kw0rFMxsvpnlJ56faWbXmtmEYEsTEUkNd3/X9Mtbm3ll214ufr9OQ+1vuBev/RKoNrPDgX8HHgV+DpwdVGEiIqmwYVcr5932PF29UfJyssjNziISdUrzc/j0Yp2G2t9wQyHm7hEzOx+41d1/bGYvD/UCM7sLOAfY7e7HDLDcgB8RD5YO4BJ3//PBlS8iMrTb/rARA/76zPlEok53JEZvNMZph1dSkq9BHfob7t9Ir5ktAy4GPpGYd6CTeu8BfgLcN8jyjwMLEo/3Af+a+FNEJCXqGtt57NUdXHn6YXztY0eGXU5GGG5H86XAqcDN7r7ZzOYBDwz1And/DmgaYpVzgfs8bgUwwcymDbMeEZEDuv3ZTeRkZ3H5B+aFXUrGGNaegruvA64FMLMKoNTdbxnhtmcA25Km6xPz3h7h+4qIsLOli1/W1nPByTOZXDZ2RzVNteGeffSMmZWZ2UTgVeBuM/vnEW57oC5/H2AeZnaVmdWYWU1DQ8MINysi48Gdf9xE1J0vnTE/7FIyynAPH5W7+z7gU8Dd7r4Y+IsRbrsemJU0PRPYMdCK7r7c3avdvbqqSucUi8jQmtt7+NnKrZx7/HRmTSwKu5yMMtxQyEkc778A+G2Ktv0o8EWLOwVocXcdOhKREbv7hS19d1CTgzPcs49uBJ4Ennf3VWZ2GPDmUC8wsweBM4FKM6sHbiBxxpK73w48Tvx01I3ET0m99FAaICKSrK07wj3Pb+ZjR09hwZTSsMvJOMPtaH4YeDhpehPw6QO8ZtkBljvwt8PZvojIcD2woo59XRH+5szDwy4lIw23o3mmmf3azHab2S4z+6WZ6VJAEUkr+7p6+bdn3+KMI6o4fpZG4jkUw+1TuJt4H8B04qeNPpaYJyKSNpY/u4nmjl6+/rGFYZeSsYYbClXufre7RxKPewCdBiQiaWP3vi7u/NMmPnn8dI6ZUR52ORlruKGwx8wuMrPsxOMioDHIwkREDsatT79JNOZc91HtJYzEcEPhMuKno+4kfsXxZ9DZQiKSJt5qaOM/Vm3jC++bw+xJui5hJIYVCu6+1d0/6e5V7j7Z3c8jfiGbiEjovv/kegpysrj6wzrjaKRGcue1r6asChGRQ/Ty1maeWLOTq86YT2VJftjlZLyRhIJuVyQioXJ3bnniDSpL8rjidI2EmgojCYUBB68TERktv1uzk5Wbm7j2rAUU64Y5KTHk36KZtTLwl78BhYFUJCIyDG3dEb792FoWTSvj80tmh13OmDFkKLi7Bg4RkbT0g6fWs7u1m9svWkxO9kgOekgy/U2KSMZZs72Fe1/YwhfeN5sTZ1eEXc6YolAQkYwSjTn/99evMakkX/ddDoBCQUQyyv0vbmF1fQt/f84iygtzwy5nzFF3vYhkhFjM2b63k+8/tYHTF1TyieOmhV3SmKRQEJG01NUb5dK7V7G1qYN9Xb20dUdwh7ycLG469xjMdKlUEBQKIpKWVm1p4sVNjZx15GRmTSyitCCH0oIclsybxNzK4rDLG7MUCiKSll7a3ER2lvGjZSdSogvTRo06mkUkLa3c3MQx08sUCKMs0FAws6Vmtt7MNprZNwZYPsfMnjaz1Wb2jG7xKSIQ7094ZdtelsybGHYp405goWBm2cBtwMeBRcAyM1vUb7XvA/e5+3HAjcB3gqpHRDLH6voWeiIxlsybFHYp406QewpLgI3uvsnde4BfAOf2W2cR8HTi+R8GWC4i49BLm+M3djx5rq5WHm1BhsIMYFvSdH1iXrJXgU8nnp8PlJqZfhqIjHMrNzdx5NRSJhTlhV3KuBNkKAx0EnH/EVevAz5oZi8DHwS2A5H3vJHZVWZWY2Y1DQ0Nqa9URNJGJBqjtq5Z/QkhCTIU6oFZSdMzgR3JK7j7Dnf/lLufCHwrMa+l/xu5+3J3r3b36qqqqgBLFpGwrd2xj46eqEIhJEGGwipggZnNM7M84ELg0eQVzKzSzPbX8E3grgDrEZEM8NLmJgCWzFUohCGwUHD3CHA18CTwOvCQu681sxvN7JOJ1c4E1pvZBmAKcHNQ9YhIZli5uYl5lcVMLisIu5RxKdCrQtz9ceDxfvOuT3r+CPBIkDWISOaIxZxVW5pYevTUsEsZt3RFs4ikjQ27W2np7FV/QogUCiKSNvr6ExQKoVEoiEjaWLm5ienlBcysKAy7lHFLoSAiacHdeWlzE0vmTdS9EkKkUBCRtLClsYOG1m6NdxQyhYKIpIX94x2pPyFcCgURSQsrNzcxqTiP+VW6q1qYFAoikhZq65pZPKdC/QkhUyiISOgaWrupa+ygWkNlh06hICKhq62LX5+weI76E8KmUBCR0NVsaSYvJ4tjZpSFXcq4p1AQkdDV1DVz/Mxy8nOywy5l3FMoiEioOnuirN3RokNHaUKhICKherV+L71Rp3qOOpnTgUJBREJVW9cMwGKFQlpQKIhIqGq2NDG/qpiK4rywSxEUCiISoljMqa1rplr9CWlDoSAiodnY0Ma+rgiLddFa2lAoiEhoarbE+xPUyZw+Ag0FM1tqZuvNbKOZfWOA5bPN7A9m9rKZrTazs4OsR0TSS01dfBC8eZUaBC9dBBYKZpYN3AZ8HFgELDOzRf1W+zvgIXc/EbgQ+GlQ9YhI+qmta+YkDYKXVoLcU1gCbHT3Te7eA/wCOLffOg7sv669HNgRYD0ikkb6BsHToaO0EmQozAC2JU3XJ+Yl+zZwkZnVA48D1wz0RmZ2lZnVmFlNQ0NDELWKyCjbPwhe9VydeZROggyFgfYHvd/0MuAed58JnA3cb2bvqcndl7t7tbtXV1VVBVCqiIw2DYKXnoIMhXpgVtL0TN57eOhy4CEAd38RKAAqA6xJRNKEBsFLTzkBvvcqYIGZzQO2E+9I/ny/dbYCZwH3mNlRxENBx4dExpi27ghPrd3JlsYO6hrbqWvsYHX9Xq46Y37YpUk/gYWCu0fM7GrgSSAbuMvd15rZjUCNuz8K/G/gDjP7CvFDS5e4e/9DTCKSwaIx54p7V7FiUxNZBtMnFDJnUhHLlszmolNmh12e9BPkngLu/jjxDuTkedcnPV8HnBZkDSISrjv+uIkVm5q46dyjueDkWTpclOYCDQURGd/WbG/hB0+tZ+nRU7nolDm6HiEDaJgLEQlEZ0+UL//HK1QU5fGdTx2rQMgQ2lMQkUDc8sTrbNzdxv2XL9Gw2BlEewoiknJ/WL+be1+s47LT5nH6Al1blEkUCiKSUo1t3Xzt4dUsnFLK15cuDLscOUg6fCQiKePufPNXr7Gvs5f7L19CQa7ONMo02lMQkZR5uKaep9bt4msfW8hR0zR8RSZSKIhISmxt7OAfHlvLKYdN5PIPzAu7HDlECgURGbFINMZXHnqFrCzjBxecQFaWTj/NVOpTEJERu/3Zt6ita+bWz53AjAmFYZcjI6A9BREZkRc27uHW37/JOcdN49wTpoddjoyQ9hRE5JC0d0f43pPruffFLcyZWMT/O+8YXbU8BigUROSgPb9xD//nl6upb+7k4lPn8PWlR1Kcr6+TsUCfoogMWyzm3PDoWu5fUce8ymIe+tKpLJmn22mOJQoFERm27zzxOveviA9f8fWlC3Vx2hikUBCRYbnvxS3c8cfNXHzqHP7+nKPUfzBG6ewjETmgp1/fxbcfXctZR07m+k8crUAYwxQKIjKkNdtbuObBl1k0vYx/WXYi2bowbUwLNBTMbKmZrTezjWb2jQGW/9DMXkk8NpjZ3iDrEZHhc3dq65q47J5VTCjM5a6LT9YZRuNAYJ+wmWUDtwEfAeqBVWb2aOK+zAC4+1eS1r8GODGoekRkeHa2dPGrl+t5pLaeTQ3tlBfmct+XTmFyWUHYpckoCDL2lwAb3X0TgJn9AjgXWDfI+suAGwKsR0SGsLOli28/upan1u0k5nDy3Aq+dMZhnH3sNEoLcsMuT0ZJkKEwA9iWNF0PvG+gFc1sDjAP+J8A6xGRAbg7v3llB9f/Zg090Rh/9cH5fLZ6FvMqi8MuTUIQZCgM1Bvlg6x7IfCIu0cHfCOzq4CrAGbPnp2a6kSExrZu/u4/1/DEmp2cNHsCP7jgBIXBOBdkKNQDs5KmZwI7Bln3QuBvB3sjd18OLAeorq4eLFhEZJj2dvTwSG09tz/7Fvs6I3zj40dy5emH6cwiCTQUVgELzGwesJ34F//n+69kZguBCuDFAGsRGffcnVe27eWBFVv57eoddEdiLJk3kZvOPYaFU0vDLk/SRGCh4O4RM7saeBLIBu5y97VmdiNQ4+6PJlZdBvzC3bUHIJJi2/d2smpzEy9taWLlpkbeaminOC+bz1bP5Avvm6NbZsp7WKZ9F1dXV3tNTU3YZYikrW1NHTz40lZ+88oOtu/tBKA0P4eT5lTwkUVTOO/EGZToeoNxx8xq3b36QOvpX4ZIhmnp7GXLnnbycrIozsuhOD+bwrxsnt/YyM9W1vHshgYM+NDCyVx5+jxOnjeRI6eWqb9AhkWhIJJm2rsjNHf00NLZS0tnL/s6e9m8p4M121tYs6OFusaOQV87uTSfaz68gGVLZjGtXLfFlIOnUBAJWVt3hJWbGnl+YyMvvLWHN3a2DrjerImFHDO9nAuqZ7FgcgnRmNPWHaGjJ0pbd4T5VSWcddRkcrM1pJkcOoWCSAA6e6K8ubuVHXs72dcZYV9X/Bf/vq4ILZ297O3oYW9nL3s7etna1EE05uTlZHHy3Aq++pEjmFpWQFlhLmWFOZQX5jJjQiETivLCbpaMAwoFkSH0RmPs7Yh/iXdHYrhD1J1ozOmORONf9Ikv/ab2HjbubmPDrlbqmjrofw6HGZTk5zChKJcJhXlMKIp/2Z997FROm1/JSXMqdNMaCZ1CQYT4L/vV9Xup3drMn+ua2bi7jab2HvZ1RYb9HtlZxtxJRSyaXsZ5J87gyKmlzJpYRHlhLmWFuZTk5ZClzl5JcwoFGVdiMWd3azfrd7WyYWcr63e18sbOfbzxdiuRWPyn/WFVxRwzo5zKknwqivKYWJxLeVEeedlZZGcZ2VmQZUZeThblhbn60pcxRaEgGScac7Y0trNhZys90Rj5OVnk52STn5NFzGHnvi52tnTydksXO1u6aOro6TsE1NLZSyzpsE5VaT4Lp5Ry1RmHsXhOBSfOrmBisY7dy/ilUJDAuTuN7T28uauNjbtb2bi7jZ5oLHGOfQ6lBTnkZmfR0tlLc+ILvLmjB4CCnPg5+AW5WfRGnQ27Wlm/s5XuSOyA251YnMeUsgIqS/KYMaGQiqL4cfyq0nyOmFLKEVNKFQAi/SgU5IAi0RitXfHO1CwzKkvyKcwbukN0x95OntvQwLMbGli5uYmm9p6+ZSX5ORTmZdPWFaGz990D4+7viK0oysMMunqjdPXG6OyNkmWwYHIpf3nKHI6cVsbCKaUU5WfT3RujOxKlOxLDgKnlBUwpK1CnrcghUCgEwN3fOQ2xK352SmtXL4V52Ylj1PHHoX5pxWJOe0+E1q74ozsSTRzrNnKyjJysd451D3SM293p7I2ytamDLXs62NLYTl1jOw2t3bR3R+noidDWHaG9O0prVy/tPe8d0bwoL5tJJXlMLIq3Iz83fvgmLyeLDTtbeXN3GwDTygs468jJHDWtjAVTSlgwuZQpZfl9N36PJtrSE4lRVpBLXo7OsRcJ07gJhUg0Rlt3/JdpV2+Mzp4oXZEok4rzmFVRdMgdhO7xjss121tYXd/C6vq9vLa9hT1tPQd8bWl+DrMnFTF3UjFzJhUxe2IRkZjHD6O0x89j339F676ueLDs6+ylrTvyruPig8kymFCUR0VRLrnZWbT3RGjrin/h90bf/QaTivOYXFZASX42FcV5zKwooigvO36ufEH8fPnSglxiMWdPezeNbT00tnXT2B4/VbOls5fu3ig9kRjTJxRyQfUsPriwigWTS/oCYCDZWUaZ7uolkjbGTSg8vmYn1z748oDLivOyWTi1lKOmlbFwaimzJ8a/oGdUFJKfE/8139Ydob65g/qmTuqaOti4u5U3d7Xx5u42Wjp7gfiX8OGTSzhz4WQWTillQlEupfu/UPNz6eyN0tTeQ3NHD03tPeze10VdUwfr3t7Hk2t39p39AlCYm01FUW7iAqZcZkwooKyglNKCHMoKc+N/FsTfPz8ni6g7sZgTiTmRWPzc+ub2HhoT2+uJOGUFOZQU5FCSH/+Cn1lRGA+kyiJ9MYsIMI5C4dgZ5Vx/ziIKcrMpzMuiMDeb/Jxsdu3r4vW39/H6zlYefXUHrSvfOS/dDKaUFtAdidLc0fuu96soymXBlFLOOW4ah08u4ejp5Rw9vYziQxx9MhKN8XZLV99pjjoeLiJhGDehMK+ymHkfmDfkOu7Orn3dbGvuYGtjB9uaO9jW1ElhXhYzK4qYWVHIzIoiZlUUMqkkP6X15WRnMWtiUUrfU0TkYI2bUBgOM2NqeQFTyws4ee7EsMsRERl1OtVDRET6KBRERKSPQkFERPoEGgpmttTM1pvZRjP7xiDrXGBm68xsrZn9PMh6RERkaIF1NJtZNnAb8BGgHlhlZo+6+7qkdRYA3wROc/dmM5scVD0iInJgQe4pLAE2uvsmd+8BfgGc22+dK4Hb3L0ZwN13B1iPiIgcQJChMAPYljRdn5iX7AjgCDN73sxWmNnSgd7IzK4ysxozq2loaAioXBERCTIUBhrwpv+IPTnAAuBMYBlwp5lNeM+L3Je7e7W7V1dVVaW8UBERiQvy4rV6YFbS9ExgxwDrrHD3XmCzma0nHhKrBnvT2traPWZWlzSrHGgZYNX+84eaHux5JbBnsFqGYbDaDmY9te/A0+OxfSNt21C1Hcx6at+Bp9OlfXOG9a7uHsiDeOBsAuYBecCrwNH91lkK3Jt4Xkn8cNOkg9zO8uHMH2p6iOc1I/w7GLC2g1lP7VP7Bno+0rapfWrfYI/ADh+5ewS4GngSeB14yN3XmtmNZvbJxGpPAo1mtg74A/A1d288yE09Nsz5Q00P9nykhvteQ62n9h14Wu07NGrfgdcb6+17D0skiQzAzGrcvTrsOoKi9mWusdw2UPvCpCuah7Y87AICpvZlrrHcNlD7QqM9BRER6aM9BRER6TMuQsHM7jKz3Wa25hBeu9jMXkuM3/QvlnTDYTO7JjG201oz+6fUVn1QNaa8fWb2bTPbbmavJB5np77yYdcYyOeXWH6dmbmZVaau4oOuMYjP7yYzW5347J4ys+mpr3zYNQbRvu+Z2RuJNv56oOubRktA7fts4nslZmaj2/cw0tOiMuEBnAGcBKw5hNe+BJxK/GK8J4CPJ+Z/CPg9kJ+YnjzG2vdt4LqwP7ug2pdYNov4GXB1QOVYah9QlrTOtcDtY6x9HwVyEs+/C3x3jLXvKGAh8AxQPZrtGRd7Cu7+HNCUPM/M5pvZ78ys1sz+aGZH9n+dmU0j/p/rRY9/UvcB5yUW/zVwi7t3J7YR2rhNAbUvbQTYvh8CX+e9V9qPqiDa5+77klYtJsQ2BtS+pzx+2jvACuIXx4YioPa97u7rR6P+/sZFKAxiOXCNuy8GrgN+OsA6M4hfdb1f8vhNRwCnm9lKM3vWzE4OtNqDN9L2AVyd2D2/y8wqgiv1kIyofRa/Vma7u78adKGHaMSfn5ndbGbbgC8A1wdY66FIxb/P/S4j/is7naSyfaNqXN6j2cy3v861AAAEz0lEQVRKgPcDDycdYs4faNUB5u3/xZUDVACnACcDD5nZYYnED1WK2vevwE2J6ZuAHxD/zxe6kbbPzIqAbxE/BJF2UvT54e7fAr5lZt8kfiHpDSku9ZCkqn2J9/oWEAF+lsoaRyKV7QvDuAwF4ntIe939hOSZFr8HRG1i8lHiX4zJu6XJ4zfVA79KhMBLZhYjPlRHOgzjOuL2ufuupNfdAfw2yIIP0kjbN5/48CuvJv7TzgT+bGZL3H1nwLUPRyr+fSb7OfBfpEkokKL2mdnFwDnAWenwYyxJqj+/0RVW58xoP4C5JHUEAS8An008N+D4QV63ivjewP6OoLMT8/8KuDHx/Aji4zbZGGrftKR1vgL8Yix9fv3W2UKIHc0BfX4Lkta5BnhkjLVvKbAOqAqzXUH/+ySEjubQ/zJH6QN7EHgb6CX+C/9y4r8Uf0d8oL51wPWDvLYaWAO8Bfxk/xc/8UH+Hkgs+zPw4THWvvuB14DVxH/VTBut9oxG+/qtE2ooBPT5/TIxfzXxsW9mjLH2bST+Q+yVxCPMs6uCaN/5iffqBnYBT45We3RFs4iI9BnPZx+JiEg/CgUREemjUBARkT4KBRER6aNQEBGRPgoFGRPMrG2Ut3enmS1K0XtFE6OZrjGzxw404qeZTTCzv0nFtkX60ympMiaYWZu7l6Tw/XL8nQHXApVcu5ndC2xw95uHWH8u8Ft3P2Y06pPxRXsKMmaZWZWZ/dLMViUepyXmLzGzF8zs5cSfCxPzLzGzh83sMeApMzvTzJ4xs0cSY/f/LGm8+2f2j3NvZm2JwedeNbMVZjYlMX9+YnqVmd04zL2ZF3ln0L4SM3vazP5s8TH3z02scwswP7F38b3Eul9LbGe1mf1DCv8aZZxRKMhY9iPgh+5+MvBp4M7E/DeAM9z9ROKjh/5j0mtOBS529w8npk8EvgwsAg4DThtgO8XACnc/HngOuDJp+z9KbP+AY9okxsY5i/gV5ABdwPnufhLx+3f8IBFK3wDecvcT3P1rZvZRYAGwBDgBWGxmZxxoeyIDGa8D4sn48BfAoqSRKsvMrBQoB+41swXER6XMTXrNf7t78tj4L7l7PYCZvUJ8jJs/9dtOD+8MGFgLfCTx/FTeuX/Dz4HvD1JnYdJ71wL/nZhvwD8mvuBjxPcgpgzw+o8mHi8npkuIh8Rzg2xPZFAKBRnLsoBT3b0zeaaZ/Rj4g7ufnzg+/0zS4vZ+79Gd9DzKwP9nev2dzrnB1hlKp7ufYGblxMPlb4F/IX4fhCpgsbv3mtkWoGCA1xvwHXf/t4Pcrsh76PCRjGVPEb+PAABmtn8o43Jge+L5JQFufwXxw1YAFx5oZXdvIX7rzOvMLJd4nbsTgfAhYE5i1VagNOmlTwKXJcbxx8xmmNnkFLVBxhmFgowVRWZWn/T4KvEv2OpE5+s64sOdA/wT8B0zex7IDrCmLwNfNbOXgGlAy4Fe4O4vEx9Z80LiN46pNrMa4nsNbyTWaQSeT5zC+j13f4r44akXzew14BHeHRoiw6ZTUkUCkrjDW6e7u5ldCCxz93MP9DqRMKlPQSQ4i4GfJM4Y2kua3M5UZCjaUxARkT7qUxARkT4KBRER6aNQEBGRPgoFERHpo1AQEZE+CgUREenz/wGhR01FsrUokQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 2.0e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='68' class='' max='120', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      56.67% [68/120 6:26:40<4:55:41]\n",
       "    </div>\n",
       "    \n",
       "<table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>0.647744</th>\n",
       "    <th>0.560832</th>\n",
       "    <th>0.800000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>0.642336</th>\n",
       "    <th>0.563320</th>\n",
       "    <th>0.786667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>0.636642</th>\n",
       "    <th>0.557776</th>\n",
       "    <th>0.800000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>0.631460</th>\n",
       "    <th>0.553656</th>\n",
       "    <th>0.786667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>5</th>\n",
       "    <th>0.628575</th>\n",
       "    <th>0.554631</th>\n",
       "    <th>0.753333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>6</th>\n",
       "    <th>0.624284</th>\n",
       "    <th>0.536982</th>\n",
       "    <th>0.740000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>7</th>\n",
       "    <th>0.619385</th>\n",
       "    <th>0.552092</th>\n",
       "    <th>0.740000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>8</th>\n",
       "    <th>0.614203</th>\n",
       "    <th>0.542687</th>\n",
       "    <th>0.733333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>9</th>\n",
       "    <th>0.606767</th>\n",
       "    <th>0.546605</th>\n",
       "    <th>0.740000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>10</th>\n",
       "    <th>0.601990</th>\n",
       "    <th>0.544118</th>\n",
       "    <th>0.740000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>11</th>\n",
       "    <th>0.597472</th>\n",
       "    <th>0.516561</th>\n",
       "    <th>0.740000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>12</th>\n",
       "    <th>0.592147</th>\n",
       "    <th>0.532537</th>\n",
       "    <th>0.680000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>13</th>\n",
       "    <th>0.585171</th>\n",
       "    <th>0.543683</th>\n",
       "    <th>0.686667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>14</th>\n",
       "    <th>0.579484</th>\n",
       "    <th>0.502827</th>\n",
       "    <th>0.726667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>15</th>\n",
       "    <th>0.577527</th>\n",
       "    <th>0.551526</th>\n",
       "    <th>0.700000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>16</th>\n",
       "    <th>0.576576</th>\n",
       "    <th>0.583993</th>\n",
       "    <th>0.620000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>17</th>\n",
       "    <th>0.575512</th>\n",
       "    <th>0.566347</th>\n",
       "    <th>0.680000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>18</th>\n",
       "    <th>0.572978</th>\n",
       "    <th>0.576792</th>\n",
       "    <th>0.653333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>19</th>\n",
       "    <th>0.565390</th>\n",
       "    <th>0.555535</th>\n",
       "    <th>0.706667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>20</th>\n",
       "    <th>0.551884</th>\n",
       "    <th>0.532342</th>\n",
       "    <th>0.706667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>21</th>\n",
       "    <th>0.554074</th>\n",
       "    <th>0.544294</th>\n",
       "    <th>0.726667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>22</th>\n",
       "    <th>0.558102</th>\n",
       "    <th>0.576139</th>\n",
       "    <th>0.673333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>23</th>\n",
       "    <th>0.550744</th>\n",
       "    <th>0.533298</th>\n",
       "    <th>0.720000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>24</th>\n",
       "    <th>0.554532</th>\n",
       "    <th>0.480118</th>\n",
       "    <th>0.766667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>25</th>\n",
       "    <th>0.551981</th>\n",
       "    <th>0.509561</th>\n",
       "    <th>0.720000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>26</th>\n",
       "    <th>0.539476</th>\n",
       "    <th>0.512149</th>\n",
       "    <th>0.740000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>27</th>\n",
       "    <th>0.544763</th>\n",
       "    <th>0.524027</th>\n",
       "    <th>0.726667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>28</th>\n",
       "    <th>0.547540</th>\n",
       "    <th>0.541455</th>\n",
       "    <th>0.740000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>29</th>\n",
       "    <th>0.532257</th>\n",
       "    <th>0.486601</th>\n",
       "    <th>0.800000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>30</th>\n",
       "    <th>0.528153</th>\n",
       "    <th>0.547931</th>\n",
       "    <th>0.740000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>31</th>\n",
       "    <th>0.532592</th>\n",
       "    <th>0.549040</th>\n",
       "    <th>0.700000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>32</th>\n",
       "    <th>0.529408</th>\n",
       "    <th>0.424521</th>\n",
       "    <th>0.826667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>33</th>\n",
       "    <th>0.526248</th>\n",
       "    <th>0.472475</th>\n",
       "    <th>0.766667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>34</th>\n",
       "    <th>0.529958</th>\n",
       "    <th>0.556221</th>\n",
       "    <th>0.760000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>35</th>\n",
       "    <th>0.515804</th>\n",
       "    <th>0.575344</th>\n",
       "    <th>0.733333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>36</th>\n",
       "    <th>0.510442</th>\n",
       "    <th>0.587654</th>\n",
       "    <th>0.706667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>37</th>\n",
       "    <th>0.516505</th>\n",
       "    <th>0.477186</th>\n",
       "    <th>0.746667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>38</th>\n",
       "    <th>0.507994</th>\n",
       "    <th>0.507344</th>\n",
       "    <th>0.753333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>39</th>\n",
       "    <th>0.504094</th>\n",
       "    <th>0.540656</th>\n",
       "    <th>0.753333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>40</th>\n",
       "    <th>0.499890</th>\n",
       "    <th>0.531011</th>\n",
       "    <th>0.720000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>41</th>\n",
       "    <th>0.502131</th>\n",
       "    <th>0.491928</th>\n",
       "    <th>0.753333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>42</th>\n",
       "    <th>0.491473</th>\n",
       "    <th>0.542035</th>\n",
       "    <th>0.753333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>43</th>\n",
       "    <th>0.493651</th>\n",
       "    <th>0.594480</th>\n",
       "    <th>0.693333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>44</th>\n",
       "    <th>0.492904</th>\n",
       "    <th>0.436231</th>\n",
       "    <th>0.813333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>45</th>\n",
       "    <th>0.492346</th>\n",
       "    <th>0.547132</th>\n",
       "    <th>0.726667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>46</th>\n",
       "    <th>0.484081</th>\n",
       "    <th>0.439894</th>\n",
       "    <th>0.780000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>47</th>\n",
       "    <th>0.485249</th>\n",
       "    <th>0.567602</th>\n",
       "    <th>0.693333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>48</th>\n",
       "    <th>0.477533</th>\n",
       "    <th>0.508393</th>\n",
       "    <th>0.766667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>49</th>\n",
       "    <th>0.479224</th>\n",
       "    <th>0.541249</th>\n",
       "    <th>0.766667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>50</th>\n",
       "    <th>0.470968</th>\n",
       "    <th>0.509807</th>\n",
       "    <th>0.800000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>51</th>\n",
       "    <th>0.455762</th>\n",
       "    <th>0.503552</th>\n",
       "    <th>0.753333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>52</th>\n",
       "    <th>0.449086</th>\n",
       "    <th>0.502987</th>\n",
       "    <th>0.766667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>53</th>\n",
       "    <th>0.440173</th>\n",
       "    <th>0.484018</th>\n",
       "    <th>0.773333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>54</th>\n",
       "    <th>0.444516</th>\n",
       "    <th>0.553979</th>\n",
       "    <th>0.726667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>55</th>\n",
       "    <th>0.436846</th>\n",
       "    <th>0.578157</th>\n",
       "    <th>0.760000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>56</th>\n",
       "    <th>0.433742</th>\n",
       "    <th>0.469290</th>\n",
       "    <th>0.806667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>57</th>\n",
       "    <th>0.430673</th>\n",
       "    <th>0.518790</th>\n",
       "    <th>0.753333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>58</th>\n",
       "    <th>0.428197</th>\n",
       "    <th>0.483164</th>\n",
       "    <th>0.760000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>59</th>\n",
       "    <th>0.426662</th>\n",
       "    <th>0.491882</th>\n",
       "    <th>0.753333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>60</th>\n",
       "    <th>0.424013</th>\n",
       "    <th>0.659668</th>\n",
       "    <th>0.713333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>61</th>\n",
       "    <th>0.428552</th>\n",
       "    <th>0.562455</th>\n",
       "    <th>0.740000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>62</th>\n",
       "    <th>0.413376</th>\n",
       "    <th>0.474612</th>\n",
       "    <th>0.793333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>63</th>\n",
       "    <th>0.411091</th>\n",
       "    <th>0.437474</th>\n",
       "    <th>0.833333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>64</th>\n",
       "    <th>0.409153</th>\n",
       "    <th>0.410082</th>\n",
       "    <th>0.826667</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>65</th>\n",
       "    <th>0.404515</th>\n",
       "    <th>0.504992</th>\n",
       "    <th>0.760000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>66</th>\n",
       "    <th>0.400810</th>\n",
       "    <th>0.448490</th>\n",
       "    <th>0.800000</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>67</th>\n",
       "    <th>0.394757</th>\n",
       "    <th>0.493412</th>\n",
       "    <th>0.813333</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>68</th>\n",
       "    <th>0.390002</th>\n",
       "    <th>0.469705</th>\n",
       "    <th>0.813333</th>\n",
       "  </tr>\n",
       "</table>\n",
       "\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='4' class='' max='43', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      9.30% [4/43 00:28<04:40 0.3907]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(120, max_lr=lr, wd=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('vgg16like-avgpool-exact-relu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Estimate AUC on testing set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "data_test = ImageDataBunch.from_folder(path, train=\"train\", valid=\"test\", valid_pct=None,dl_tfms=None, size=sz, num_workers=4).normalize(imagenet_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['melanoma', 'other'], 2, 2800)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_test.classes, data_test.c, len(data_test.train_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_inf = Learner(data_test, model, loss_func=nn.CrossEntropyLoss(), metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Learner(data=ImageDataBunch;\n",
       "\n",
       "Train: LabelList\n",
       "y: CategoryList (2800 items)\n",
       "[Category other, Category other, Category other, Category other, Category other]...\n",
       "Path: data\n",
       "x: ImageItemList (2800 items)\n",
       "[Image (3, 4361, 6602), Image (3, 2016, 3024), Image (3, 768, 1024), Image (3, 768, 1024), Image (3, 768, 1024)]...\n",
       "Path: data;\n",
       "\n",
       "Valid: LabelList\n",
       "y: CategoryList (600 items)\n",
       "[Category other, Category other, Category other, Category other, Category other]...\n",
       "Path: data\n",
       "x: ImageItemList (600 items)\n",
       "[Image (3, 2000, 3008), Image (3, 4439, 6688), Image (3, 2000, 3008), Image (3, 2848, 4288), Image (3, 2000, 3008)]...\n",
       "Path: data;\n",
       "\n",
       "Test: None, model=vgg16Like(\n",
       "  (block1): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  )\n",
       "  (block2): Sequential(\n",
       "    (0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  )\n",
       "  (block3): Sequential(\n",
       "    (0): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (4): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (7): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (8): ReLU()\n",
       "    (9): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  )\n",
       "  (block4): Sequential(\n",
       "    (0): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (7): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (8): ReLU()\n",
       "    (9): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  )\n",
       "  (block5): Sequential(\n",
       "    (0): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "    (3): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (4): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU()\n",
       "    (6): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "    (7): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (8): ReLU()\n",
       "    (9): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  )\n",
       "  (dense): Sequential(\n",
       "    (0): Linear(in_features=18432, out_features=2, bias=True)\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=CrossEntropyLoss(), metrics=[<function accuracy at 0x7f0d12556ea0>], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('data'), model_dir='models', callback_fns=[<class 'fastai.basic_train.Recorder'>], callbacks=[], layer_groups=[Sequential(\n",
       "  (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (2): ReLU()\n",
       "  (3): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (5): ReLU()\n",
       "  (6): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  (7): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (8): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (9): ReLU()\n",
       "  (10): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (12): ReLU()\n",
       "  (13): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  (14): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (15): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (16): ReLU()\n",
       "  (17): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (19): ReLU()\n",
       "  (20): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (21): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (22): ReLU()\n",
       "  (23): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  (24): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (25): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (26): ReLU()\n",
       "  (27): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (28): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (29): ReLU()\n",
       "  (30): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (31): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (32): ReLU()\n",
       "  (33): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  (34): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (35): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (36): ReLU()\n",
       "  (37): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (38): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (39): ReLU()\n",
       "  (40): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (42): ReLU()\n",
       "  (43): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  (44): Linear(in_features=18432, out_features=2, bias=True)\n",
       ")])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_inf.load('vgg16like-avgpool-exact-relu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_test, labels_test = learn_inf.get_preds(ds_type=DatasetType.Valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8883934101325407"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(labels_test, np.array(preds_test.tolist())[:,1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export to Protocol Buffer file "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pytorch_model = learn.model\n",
    "pytorch_model.eval()\n",
    "\n",
    "export_to_pb(pytorch_model, filename='skincancer_model.pb')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}