5.2. DSLR_to_WEBCAM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "use_gpu = True\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import os.path\n",
    "\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import manifold\n",
    "from math import exp, sqrt\n",
    "from torch.autograd import Variable\n",
    "from my_dataset import MNIST_M\n",
    "from my_dataset import ST_Dataset\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext skip_kernel_extension\n",
    "\n",
    "import torch.utils.model_zoo as model_zoo\n",
    "\n",
    "from data_loader import get_train_test_loader, get_office31_dataloader\n",
    "\n",
    "use_gpu = torch.cuda.is_available()\n",
    "print(\"use_gpu = \" + str(use_gpu))\n",
    "\n",
    "def reset_seq(seq):\n",
    "    for m in seq:\n",
    "        if isinstance(m, nn.Conv2d):\n",
    "            n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
    "            m.weight.data.normal_(0, sqrt(2 / n))\n",
    "            if m.bias is not None:\n",
    "                m.bias.data.zero_()\n",
    "        elif isinstance(m, nn.Linear):\n",
    "            size = m.weight.size()\n",
    "            fan_out = size[0] # number of rows\n",
    "            fan_in = size[1] # number of columns\n",
    "            m.weight.data.normal_(0, sqrt(2 / (fan_in + fan_out)))\n",
    "            m.bias.data.zero_()\n",
    "        elif hasattr(m, 'reset_parameters'):\n",
    "            m.reset_parameters()\n",
    "            \n",
    "def evaluate_da_accuracy(model, dataloader, source):\n",
    "    model.eval()\n",
    "    correct_LC = 0\n",
    "    correct_DC = 0\n",
    "    total = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for i, data in enumerate(dataloader):\n",
    "            inputs, labels = data\n",
    "            if (use_gpu):\n",
    "                inputs, labels = inputs.cuda(), labels.cuda()\n",
    "            inputs, labels = Variable(inputs), Variable(labels)\n",
    "            outputs_LC, outputs_DC = model(inputs)\n",
    "            correct_LC += (torch.max(outputs_LC.data, 1)[1] == labels.data).sum().item()\n",
    "            if source:\n",
    "                correct_DC += labels.size()[0] - outputs_DC.data.sum().item()\n",
    "            else:\n",
    "                correct_DC += outputs_DC.data.sum().item()\n",
    "            total += labels.size()[0]\n",
    "        acc_LC = correct_LC / total\n",
    "        acc_DC = correct_DC / total\n",
    "    return acc_LC, acc_DC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Loading datasets: dslr\n"
     ]
    }
   ],
   "source": [
    "trainloader_dslr = get_office31_dataloader(\"dslr\", batch_size=49)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Loading datasets: webcam\n"
     ]
    }
   ],
   "source": [
    "trainloader_webcam = get_office31_dataloader(\"webcam\", batch_size=79)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRL_func(torch.autograd.Function):\n",
    "\n",
    "    @staticmethod\n",
    "    def forward(ctx, inputs, lamda):\n",
    "        ctx.save_for_backward(lamda)\n",
    "        return inputs\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_outputs):\n",
    "        lamda, = ctx.saved_tensors\n",
    "        return -lamda * grad_outputs, None\n",
    "\n",
    "class GRL(nn.Module):\n",
    "    \n",
    "    def __init__(self, lamda_init):\n",
    "        super(GRL, self).__init__()\n",
    "        self.GRL_func = GRL_func.apply\n",
    "        self.lamda = nn.Parameter(torch.Tensor(1), requires_grad=False)\n",
    "        self.set_lamda(lamda_init)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.GRL_func(x, self.lamda)\n",
    "    \n",
    "    def set_lamda(self, lamda_new):\n",
    "        self.lamda[0] = lamda_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AlexNet_DA(nn.Module):\n",
    "\n",
    "    def __init__(self, lamda_init):\n",
    "        super(AlexNet_DA, self).__init__()\n",
    "        # lambda\n",
    "        self.lamda = lamda_init\n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "            nn.Conv2d(64, 192, kernel_size=5, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "            nn.Conv2d(192, 384, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(384, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(256, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "        )\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(256 * 6 * 6, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(4096, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(4096, 256),\n",
    "            nn.ReLU(inplace=True)\n",
    "        )\n",
    "        self.label_classifier = nn.Sequential(\n",
    "            nn.Linear(256, 31)\n",
    "        )\n",
    "        self.GRL_layer = GRL(lamda_init)\n",
    "        self.domain_classifier = nn.Sequential(\n",
    "            nn.Linear(256, 1024),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(1024, 1024),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(1024, 256),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(256, 1),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.size(0), 256 * 6 * 6)\n",
    "        x = self.classifier(x)\n",
    "        x_l = self.label_classifier(x)\n",
    "        x_d = self.GRL_layer(x)\n",
    "        x_d = self.domain_classifier(x_d)\n",
    "        return x_l, x_d\n",
    "    \n",
    "    def set_lamda(self, lamda_new):\n",
    "        self.GRL_layer.set_lamda(lamda_new)\n",
    "    \n",
    "    def load_pretrained_part(self, state_dict):\n",
    "        own_state = self.state_dict()\n",
    "        for name, param in state_dict.items():\n",
    "            if name not in own_state:\n",
    "                continue\n",
    "            if isinstance(param, nn.Parameter):\n",
    "                # backwards compatibility for serialized parameters\n",
    "                param = param.data\n",
    "            own_state[name].copy_(param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AlexNet_DA(\n",
      "  (features): Sequential(\n",
      "    (0): Conv2d(3, 64, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))\n",
      "    (1): ReLU(inplace)\n",
      "    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    (3): Conv2d(64, 192, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "    (4): ReLU(inplace)\n",
      "    (5): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "    (6): Conv2d(192, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (7): ReLU(inplace)\n",
      "    (8): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (9): ReLU(inplace)\n",
      "    (10): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (11): ReLU(inplace)\n",
      "    (12): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  )\n",
      "  (classifier): Sequential(\n",
      "    (0): Dropout(p=0.5)\n",
      "    (1): Linear(in_features=9216, out_features=4096, bias=True)\n",
      "    (2): ReLU(inplace)\n",
      "    (3): Dropout(p=0.5)\n",
      "    (4): Linear(in_features=4096, out_features=4096, bias=True)\n",
      "    (5): ReLU(inplace)\n",
      "    (6): Dropout(p=0.5)\n",
      "    (7): Linear(in_features=4096, out_features=256, bias=True)\n",
      "    (8): ReLU(inplace)\n",
      "  )\n",
      "  (label_classifier): Sequential(\n",
      "    (0): Linear(in_features=256, out_features=31, bias=True)\n",
      "  )\n",
      "  (GRL_layer): GRL()\n",
      "  (domain_classifier): Sequential(\n",
      "    (0): Linear(in_features=256, out_features=1024, bias=True)\n",
      "    (1): ReLU(inplace)\n",
      "    (2): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "    (3): ReLU(inplace)\n",
      "    (4): Linear(in_features=1024, out_features=256, bias=True)\n",
      "    (5): ReLU(inplace)\n",
      "    (6): Linear(in_features=256, out_features=1, bias=True)\n",
      "    (7): Sigmoid()\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "cnn_da = AlexNet_DA(0)\n",
    "\n",
    "model_url = \"https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth\"\n",
    "\n",
    "cnn_da.load_pretrained_part(model_zoo.load_url(model_url))\n",
    "\n",
    "if (use_gpu):\n",
    "    cnn_da.cuda()\n",
    "print(cnn_da)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "para_file_dslr = \"./parameters/cnn_dslr_to_webcam_source_only.pt\"\n",
    "load_model_dslr = os.path.isfile(para_file_dslr)\n",
    "print(load_model_dslr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for param in cnn_da.features.parameters():\n",
    "    param.requires_grad = False\n",
    "\n",
    "lr_init = 0.01\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(filter(lambda p: p.requires_grad, cnn_da.parameters()), lr=lr_init, momentum=0.9)\n",
    "\n",
    "def adjust_lr(optimizer, p):\n",
    "    global lr_init\n",
    "    lr_0 = lr_init\n",
    "    alpha = 10\n",
    "    beta = 0.75\n",
    "    lr = lr_0 / (1 + alpha * p) ** beta\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 2] batch loss: 3.625\n",
      "[ 4] batch loss: 6.135\n",
      "[ 6] batch loss: 4.907\n",
      "[ 8] batch loss: 4.585\n",
      "[10] batch loss: 4.508\n",
      "epoch 1 loss: inf -> 51.622\n",
      "\n",
      "[ 2] batch loss: 1.235\n",
      "[ 4] batch loss: 2.368\n",
      "[ 6] batch loss: 2.375\n",
      "[ 8] batch loss: 2.524\n",
      "[10] batch loss: 2.663\n",
      "epoch 2 loss: 51.622 -> 23.105\n",
      "\n",
      "[ 2] batch loss: 0.645\n",
      "[ 4] batch loss: 1.399\n",
      "[ 6] batch loss: 1.200\n",
      "[ 8] batch loss: 1.656\n",
      "[10] batch loss: 1.785\n",
      "epoch 3 loss: 23.105 -> 13.486\n",
      "\n",
      "[ 2] batch loss: 0.413\n",
      "[ 4] batch loss: 0.664\n",
      "[ 6] batch loss: 0.465\n",
      "[ 8] batch loss: 0.869\n",
      "[10] batch loss: 1.081\n",
      "epoch 4 loss: 13.486 -> 6.997\n",
      "\n",
      "[ 2] batch loss: 0.297\n",
      "[ 4] batch loss: 0.418\n",
      "[ 6] batch loss: 0.274\n",
      "[ 8] batch loss: 0.479\n",
      "[10] batch loss: 0.562\n",
      "epoch 5 loss: 6.997 -> 4.069\n",
      "\n",
      "[ 2] batch loss: 0.130\n",
      "[ 4] batch loss: 0.269\n",
      "[ 6] batch loss: 0.288\n",
      "[ 8] batch loss: 0.231\n",
      "[10] batch loss: 0.426\n",
      "epoch 6 loss: 4.069 -> 2.692\n",
      "\n",
      "[ 2] batch loss: 0.047\n",
      "[ 4] batch loss: 0.160\n",
      "[ 6] batch loss: 0.114\n",
      "[ 8] batch loss: 0.265\n",
      "[10] batch loss: 0.301\n",
      "epoch 7 loss: 2.692 -> 1.779\n",
      "\n",
      "[ 2] batch loss: 0.039\n",
      "[ 4] batch loss: 0.102\n",
      "[ 6] batch loss: 0.075\n",
      "[ 8] batch loss: 0.149\n",
      "[10] batch loss: 0.197\n",
      "epoch 8 loss: 1.779 -> 1.129\n",
      "\n",
      "[ 2] batch loss: 0.066\n",
      "[ 4] batch loss: 0.122\n",
      "[ 6] batch loss: 0.075\n",
      "[ 8] batch loss: 0.110\n",
      "[10] batch loss: 0.219\n",
      "epoch 9 loss: 1.129 -> 1.186\n",
      "\n",
      "[ 2] batch loss: 0.035\n",
      "[ 4] batch loss: 0.092\n",
      "[ 6] batch loss: 0.052\n",
      "[ 8] batch loss: 0.118\n",
      "[10] batch loss: 0.152\n",
      "epoch 10 loss: 1.186 -> 0.901\n",
      "\n"
     ]
    }
   ],
   "source": [
    "prev_loss = np.float(\"inf\")\n",
    "total_epoch = 10\n",
    "finetune_epoch_start = 20\n",
    "#reset_seq(cnn_da.classifier)\n",
    "reset_seq(cnn_da.label_classifier)\n",
    "\n",
    "early_stop_cnt = 0\n",
    "\n",
    "for epoch in range(total_epoch):\n",
    "    epoch_loss = 0.0\n",
    "    running_loss = 0.0\n",
    "    p = epoch * 1.0 / total_epoch\n",
    "    lr = adjust_lr(optimizer, p)\n",
    "    if epoch == finetune_epoch_start:\n",
    "        for param in cnn_da.features.parameters():\n",
    "            param.requires_grad = True\n",
    "        optimizer = optim.SGD(filter(lambda p: p.requires_grad, cnn_da.parameters()), lr=lr, momentum=0.9)\n",
    "    for i, data in enumerate(trainloader_dslr):\n",
    "        inputs, labels = data\n",
    "        if (use_gpu):\n",
    "            inputs, labels = inputs.cuda(), labels.cuda()\n",
    "        inputs, labels = Variable(inputs), Variable(labels)\n",
    "        optimizer.zero_grad()\n",
    "        outputs_LC, _ = cnn_da(inputs)\n",
    "        loss = criterion(outputs_LC, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        epoch_loss += loss.item()\n",
    "        running_loss += loss.item()\n",
    "        if i % 2 == 1:    # print every 50 mini-batches\n",
    "            print('[%2d] batch loss: %.3f' %\n",
    "                  (i + 1, running_loss / 2))\n",
    "            running_loss = 0.0\n",
    "    print(\"epoch %d loss: %.3f -> %.3f\\n\" % (epoch + 1, prev_loss, epoch_loss))\n",
    "    if prev_loss - epoch_loss < 0.1:\n",
    "        prev_loss = epoch_loss\n",
    "        if epoch > finetune_epoch_start:\n",
    "            early_stop_cnt += 1\n",
    "            if early_stop_cnt > 2:\n",
    "                break\n",
    "    else:\n",
    "        early_stop_cnt = 0\n",
    "        prev_loss = epoch_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.0, 0.532371352475331)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_da_accuracy(cnn_da, trainloader_dslr, source=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9144654088050315, 0.47331236773316965)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_da_accuracy(cnn_da, trainloader_webcam, source=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.save(cnn_da.state_dict(), para_file_dslr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cnn_da.load_pretrained_part(torch.load(para_file_dslr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import exp\n",
    "\n",
    "lr_init = 0.01\n",
    "criterion_LC = nn.CrossEntropyLoss()\n",
    "criterion_DC = nn.BCELoss()\n",
    "optimizer = optim.SGD(filter(lambda p: p.requires_grad, cnn_da.parameters()), lr=lr_init, momentum=0.9)\n",
    "\n",
    "def adjust_lr(optimizer, p):\n",
    "    global lr_init\n",
    "    lr_0 = lr_init\n",
    "    alpha = 10\n",
    "    beta = 0.75\n",
    "    lr = lr_0 / (1 + alpha * p) ** beta\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "    return lr\n",
    "\n",
    "def adjust_lamda(model, p):\n",
    "    gamma = 10\n",
    "    lamda = 2 / (1 + exp(- gamma * p)) - 1\n",
    "    model.set_lamda(lamda)\n",
    "    return lamda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "for param in cnn_da.features.parameters():\n",
    "    param.requires_grad = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 2] batch loss: 0.677\n",
      "[ 4] batch loss: 0.714\n",
      "[ 6] batch loss: 0.690\n",
      "[ 8] batch loss: 0.745\n",
      "[10] batch loss: 0.773\n",
      "epoch 1 loss: inf -> 7.918\n",
      "LC loss: inf -> 0.344\n",
      "DC loss: inf -> 7.574\n",
      "\n",
      "[ 2] batch loss: 0.836\n",
      "[ 4] batch loss: 0.736\n",
      "[ 6] batch loss: 0.721\n",
      "[ 8] batch loss: 0.733\n",
      "[10] batch loss: 0.724\n",
      "epoch 2 loss: 7.918 -> 8.273\n",
      "LC loss: 0.344 -> 0.195\n",
      "DC loss: 7.574 -> 8.078\n",
      "\n",
      "[ 2] batch loss: 0.650\n",
      "[ 4] batch loss: 0.669\n",
      "[ 6] batch loss: 0.631\n",
      "[ 8] batch loss: 0.658\n",
      "[10] batch loss: 0.696\n",
      "epoch 3 loss: 8.273 -> 7.642\n",
      "LC loss: 0.195 -> 0.207\n",
      "DC loss: 8.078 -> 7.435\n",
      "\n",
      "[ 2] batch loss: 0.619\n",
      "[ 4] batch loss: 0.688\n",
      "[ 6] batch loss: 0.683\n",
      "[ 8] batch loss: 0.667\n",
      "[10] batch loss: 0.905\n",
      "epoch 4 loss: 7.642 -> 7.612\n",
      "LC loss: 0.207 -> 0.709\n",
      "DC loss: 7.435 -> 6.903\n",
      "\n",
      "[ 2] batch loss: 0.583\n",
      "[ 4] batch loss: 0.703\n",
      "[ 6] batch loss: 0.655\n",
      "[ 8] batch loss: 0.782\n",
      "[10] batch loss: 2.174\n",
      "epoch 5 loss: 7.612 -> 10.313\n",
      "LC loss: 0.709 -> 3.552\n",
      "DC loss: 6.903 -> 6.761\n",
      "\n",
      "[ 2] batch loss: 0.621\n",
      "[ 4] batch loss: 0.644\n",
      "[ 6] batch loss: 0.643\n",
      "[ 8] batch loss: 0.688\n",
      "[10] batch loss: 1.884\n",
      "epoch 6 loss: 10.313 -> 9.606\n",
      "LC loss: 3.552 -> 2.727\n",
      "DC loss: 6.761 -> 6.879\n",
      "\n",
      "[ 2] batch loss: 0.570\n",
      "[ 4] batch loss: 0.640\n",
      "[ 6] batch loss: 0.642\n",
      "[ 8] batch loss: 0.766\n",
      "[10] batch loss: 1.018\n",
      "epoch 7 loss: 9.606 -> 7.774\n",
      "LC loss: 2.727 -> 1.159\n",
      "DC loss: 6.879 -> 6.615\n",
      "\n",
      "[ 2] batch loss: 0.562\n",
      "[ 4] batch loss: 0.641\n",
      "[ 6] batch loss: 0.618\n",
      "[ 8] batch loss: 0.668\n",
      "[10] batch loss: 0.722\n",
      "epoch 8 loss: 7.774 -> 6.829\n",
      "LC loss: 1.159 -> 0.467\n",
      "DC loss: 6.615 -> 6.362\n",
      "\n",
      "[ 2] batch loss: 0.545\n",
      "[ 4] batch loss: 0.621\n",
      "[ 6] batch loss: 0.577\n",
      "[ 8] batch loss: 0.607\n",
      "[10] batch loss: 0.603\n",
      "epoch 9 loss: 6.829 -> 6.208\n",
      "LC loss: 0.467 -> 0.184\n",
      "DC loss: 6.362 -> 6.024\n",
      "\n",
      "[ 2] batch loss: 0.503\n",
      "[ 4] batch loss: 0.595\n",
      "[ 6] batch loss: 0.551\n",
      "[ 8] batch loss: 0.578\n",
      "[10] batch loss: 0.556\n",
      "epoch 10 loss: 6.208 -> 5.829\n",
      "LC loss: 0.184 -> 0.100\n",
      "DC loss: 6.024 -> 5.729\n",
      "0.9169811320754717 0.6208959987328487\n",
      "\n",
      "[ 2] batch loss: 0.494\n",
      "[ 4] batch loss: 0.563\n",
      "[ 6] batch loss: 0.547\n",
      "[ 8] batch loss: 0.561\n",
      "[10] batch loss: 0.529\n",
      "epoch 11 loss: 5.829 -> 5.685\n",
      "LC loss: 0.100 -> 0.069\n",
      "DC loss: 5.729 -> 5.615\n",
      "\n",
      "[ 2] batch loss: 0.523\n",
      "[ 4] batch loss: 0.683\n",
      "[ 6] batch loss: 0.538\n",
      "[ 8] batch loss: 0.564\n",
      "[10] batch loss: 0.536\n",
      "epoch 12 loss: 5.685 -> 5.884\n",
      "LC loss: 0.069 -> 0.055\n",
      "DC loss: 5.615 -> 5.829\n",
      "\n",
      "[ 2] batch loss: 0.583\n",
      "[ 4] batch loss: 0.607\n",
      "[ 6] batch loss: 0.537\n",
      "[ 8] batch loss: 0.665\n",
      "[10] batch loss: 0.576\n",
      "epoch 13 loss: 5.884 -> 6.206\n",
      "LC loss: 0.055 -> 0.046\n",
      "DC loss: 5.829 -> 6.160\n",
      "\n",
      "[ 2] batch loss: 0.502\n",
      "[ 4] batch loss: 0.642\n",
      "[ 6] batch loss: 0.617\n",
      "[ 8] batch loss: 0.602\n",
      "[10] batch loss: 0.539\n",
      "epoch 14 loss: 6.206 -> 6.175\n",
      "LC loss: 0.046 -> 0.040\n",
      "DC loss: 6.160 -> 6.135\n",
      "\n",
      "[ 2] batch loss: 0.635\n",
      "[ 4] batch loss: 0.885\n",
      "[ 6] batch loss: 0.618\n",
      "[ 8] batch loss: 0.573\n",
      "[10] batch loss: 0.556\n",
      "epoch 15 loss: 6.175 -> 6.800\n",
      "LC loss: 0.040 -> 0.036\n",
      "DC loss: 6.135 -> 6.764\n",
      "\n",
      "[ 2] batch loss: 0.543\n",
      "[ 4] batch loss: 0.608\n",
      "[ 6] batch loss: 0.555\n",
      "[ 8] batch loss: 0.582\n",
      "[10] batch loss: 0.517\n",
      "epoch 16 loss: 6.800 -> 5.816\n",
      "LC loss: 0.036 -> 0.033\n",
      "DC loss: 6.764 -> 5.783\n",
      "\n",
      "[ 2] batch loss: 0.451\n",
      "[ 4] batch loss: 0.555\n",
      "[ 6] batch loss: 0.502\n",
      "[ 8] batch loss: 0.538\n",
      "[10] batch loss: 0.467\n",
      "epoch 17 loss: 5.816 -> 5.165\n",
      "LC loss: 0.033 -> 0.030\n",
      "DC loss: 5.783 -> 5.135\n",
      "\n",
      "[ 2] batch loss: 0.435\n",
      "[ 4] batch loss: 0.522\n",
      "[ 6] batch loss: 0.486\n",
      "[ 8] batch loss: 0.511\n",
      "[10] batch loss: 0.444\n",
      "epoch 18 loss: 5.165 -> 4.910\n",
      "LC loss: 0.030 -> 0.028\n",
      "DC loss: 5.135 -> 4.883\n",
      "\n",
      "[ 2] batch loss: 0.395\n",
      "[ 4] batch loss: 0.501\n",
      "[ 6] batch loss: 0.455\n",
      "[ 8] batch loss: 0.478\n",
      "[10] batch loss: 0.422\n",
      "epoch 19 loss: 4.910 -> 4.590\n",
      "LC loss: 0.028 -> 0.026\n",
      "DC loss: 4.883 -> 4.564\n",
      "\n",
      "[ 2] batch loss: 0.360\n",
      "[ 4] batch loss: 0.473\n",
      "[ 6] batch loss: 0.429\n",
      "[ 8] batch loss: 0.457\n",
      "[10] batch loss: 0.397\n",
      "epoch 20 loss: 4.590 -> 4.294\n",
      "LC loss: 0.026 -> 0.024\n",
      "DC loss: 4.564 -> 4.270\n",
      "0.919496855345912 0.6905526341132397\n",
      "\n",
      "[ 2] batch loss: 0.359\n",
      "[ 4] batch loss: 0.448\n",
      "[ 6] batch loss: 0.415\n",
      "[ 8] batch loss: 0.452\n",
      "[10] batch loss: 0.387\n",
      "epoch 21 loss: 4.294 -> 4.193\n",
      "LC loss: 0.024 -> 0.023\n",
      "DC loss: 4.270 -> 4.170\n",
      "\n",
      "[ 2] batch loss: 0.337\n",
      "[ 4] batch loss: 0.483\n",
      "[ 6] batch loss: 0.400\n",
      "[ 8] batch loss: 0.437\n",
      "[10] batch loss: 0.381\n",
      "epoch 22 loss: 4.193 -> 4.122\n",
      "LC loss: 0.023 -> 0.022\n",
      "DC loss: 4.170 -> 4.100\n",
      "\n",
      "[ 2] batch loss: 0.334\n",
      "[ 4] batch loss: 0.432\n",
      "[ 6] batch loss: 0.453\n",
      "[ 8] batch loss: 0.503\n",
      "[10] batch loss: 0.366\n",
      "epoch 23 loss: 4.122 -> 4.224\n",
      "LC loss: 0.022 -> 0.021\n",
      "DC loss: 4.100 -> 4.204\n",
      "\n",
      "[ 2] batch loss: 0.375\n",
      "[ 4] batch loss: 0.684\n",
      "[ 6] batch loss: 0.408\n",
      "[ 8] batch loss: 0.493\n",
      "[10] batch loss: 0.528\n",
      "epoch 24 loss: 4.224 -> 5.334\n",
      "LC loss: 0.021 -> 0.020\n",
      "DC loss: 4.204 -> 5.314\n",
      "\n",
      "[ 2] batch loss: 0.675\n",
      "[ 4] batch loss: 0.542\n",
      "[ 6] batch loss: 0.648\n",
      "[ 8] batch loss: 0.846\n",
      "[10] batch loss: 0.885\n",
      "epoch 25 loss: 5.334 -> 10.383\n",
      "LC loss: 0.020 -> 0.019\n",
      "DC loss: 5.314 -> 10.364\n",
      "\n",
      "[ 2] batch loss: 0.470\n",
      "[ 4] batch loss: 0.758\n",
      "[ 6] batch loss: 0.760\n",
      "[ 8] batch loss: 0.642\n",
      "[10] batch loss: 0.605\n",
      "epoch 26 loss: 10.383 -> 6.840\n",
      "LC loss: 0.019 -> 0.018\n",
      "DC loss: 10.364 -> 6.821\n",
      "\n",
      "[ 2] batch loss: 0.496\n",
      "[ 4] batch loss: 0.578\n",
      "[ 6] batch loss: 0.551\n",
      "[ 8] batch loss: 0.572\n",
      "[10] batch loss: 0.544\n",
      "epoch 27 loss: 6.840 -> 5.790\n",
      "LC loss: 0.018 -> 0.018\n",
      "DC loss: 6.821 -> 5.772\n",
      "\n",
      "[ 2] batch loss: 0.472\n",
      "[ 4] batch loss: 0.567\n",
      "[ 6] batch loss: 0.522\n",
      "[ 8] batch loss: 0.533\n",
      "[10] batch loss: 0.521\n",
      "epoch 28 loss: 5.790 -> 5.513\n",
      "LC loss: 0.018 -> 0.017\n",
      "DC loss: 5.772 -> 5.496\n",
      "\n",
      "[ 2] batch loss: 0.440\n",
      "[ 4] batch loss: 0.539\n",
      "[ 6] batch loss: 0.502\n",
      "[ 8] batch loss: 0.504\n",
      "[10] batch loss: 0.486\n",
      "epoch 29 loss: 5.513 -> 5.159\n",
      "LC loss: 0.017 -> 0.017\n",
      "DC loss: 5.496 -> 5.142\n",
      "\n",
      "[ 2] batch loss: 0.402\n",
      "[ 4] batch loss: 0.514\n",
      "[ 6] batch loss: 0.475\n",
      "[ 8] batch loss: 0.481\n",
      "[10] batch loss: 0.461\n",
      "epoch 30 loss: 5.159 -> 4.836\n",
      "LC loss: 0.017 -> 0.016\n",
      "DC loss: 5.142 -> 4.820\n",
      "0.9220125786163522 0.6734111983821077\n",
      "\n",
      "[ 2] batch loss: 0.377\n",
      "[ 4] batch loss: 0.491\n",
      "[ 6] batch loss: 0.453\n",
      "[ 8] batch loss: 0.459\n",
      "[10] batch loss: 0.436\n",
      "epoch 31 loss: 4.836 -> 4.569\n",
      "LC loss: 0.016 -> 0.016\n",
      "DC loss: 4.820 -> 4.553\n",
      "\n",
      "[ 2] batch loss: 0.348\n",
      "[ 4] batch loss: 0.467\n",
      "[ 6] batch loss: 0.427\n",
      "[ 8] batch loss: 0.438\n",
      "[10] batch loss: 0.413\n",
      "epoch 32 loss: 4.569 -> 4.299\n",
      "LC loss: 0.016 -> 0.015\n",
      "DC loss: 4.553 -> 4.283\n",
      "\n",
      "[ 2] batch loss: 0.329\n",
      "[ 4] batch loss: 0.444\n",
      "[ 6] batch loss: 0.405\n",
      "[ 8] batch loss: 0.420\n",
      "[10] batch loss: 0.392\n",
      "epoch 33 loss: 4.299 -> 4.070\n",
      "LC loss: 0.015 -> 0.015\n",
      "DC loss: 4.283 -> 4.055\n",
      "\n",
      "[ 2] batch loss: 0.306\n",
      "[ 4] batch loss: 0.423\n",
      "[ 6] batch loss: 0.382\n",
      "[ 8] batch loss: 0.399\n",
      "[10] batch loss: 0.372\n",
      "epoch 34 loss: 4.070 -> 3.842\n",
      "LC loss: 0.015 -> 0.014\n",
      "DC loss: 4.055 -> 3.828\n",
      "\n",
      "[ 2] batch loss: 0.287\n",
      "[ 4] batch loss: 0.404\n",
      "[ 6] batch loss: 0.360\n",
      "[ 8] batch loss: 0.381\n",
      "[10] batch loss: 0.355\n",
      "epoch 35 loss: 3.842 -> 3.635\n",
      "LC loss: 0.014 -> 0.014\n",
      "DC loss: 3.828 -> 3.621\n",
      "\n",
      "[ 2] batch loss: 0.271\n",
      "[ 4] batch loss: 0.387\n",
      "[ 6] batch loss: 0.342\n",
      "[ 8] batch loss: 0.367\n",
      "[10] batch loss: 0.339\n",
      "epoch 36 loss: 3.635 -> 3.463\n",
      "LC loss: 0.014 -> 0.014\n",
      "DC loss: 3.621 -> 3.449\n",
      "\n",
      "[ 2] batch loss: 0.258\n",
      "[ 4] batch loss: 0.375\n",
      "[ 6] batch loss: 0.326\n",
      "[ 8] batch loss: 0.353\n",
      "[10] batch loss: 0.325\n",
      "epoch 37 loss: 3.463 -> 3.322\n",
      "LC loss: 0.014 -> 0.013\n",
      "DC loss: 3.449 -> 3.309\n",
      "\n",
      "[ 2] batch loss: 0.240\n",
      "[ 4] batch loss: 0.355\n",
      "[ 6] batch loss: 0.315\n",
      "[ 8] batch loss: 0.335\n",
      "[10] batch loss: 0.311\n",
      "epoch 38 loss: 3.322 -> 3.148\n",
      "LC loss: 0.013 -> 0.013\n",
      "DC loss: 3.309 -> 3.135\n",
      "\n",
      "[ 2] batch loss: 0.231\n",
      "[ 4] batch loss: 0.365\n",
      "[ 6] batch loss: 0.289\n",
      "[ 8] batch loss: 0.344\n",
      "[10] batch loss: 0.313\n",
      "epoch 39 loss: 3.148 -> 3.128\n",
      "LC loss: 0.013 -> 0.013\n",
      "DC loss: 3.135 -> 3.115\n",
      "\n",
      "[ 2] batch loss: 0.233\n",
      "[ 4] batch loss: 0.341\n",
      "[ 6] batch loss: 0.342\n",
      "[ 8] batch loss: 0.372\n",
      "[10] batch loss: 0.296\n",
      "epoch 40 loss: 3.128 -> 3.196\n",
      "LC loss: 0.013 -> 0.012\n",
      "DC loss: 3.115 -> 3.184\n",
      "0.9207547169811321 0.5856606573428748\n",
      "\n",
      "[ 2] batch loss: 0.346\n",
      "[ 4] batch loss: 0.637\n",
      "[ 6] batch loss: 0.347\n",
      "[ 8] batch loss: 0.681\n",
      "[10] batch loss: 0.470\n",
      "epoch 41 loss: 3.196 -> 5.407\n",
      "LC loss: 0.012 -> 0.012\n",
      "DC loss: 3.184 -> 5.395\n",
      "\n",
      "[ 2] batch loss: 0.320\n",
      "[ 4] batch loss: 0.651\n",
      "[ 6] batch loss: 1.043\n",
      "[ 8] batch loss: 0.645\n",
      "[10] batch loss: 0.377\n",
      "epoch 42 loss: 5.407 -> 6.443\n",
      "LC loss: 0.012 -> 0.012\n",
      "DC loss: 5.395 -> 6.431\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 2] batch loss: 1.004\n",
      "[ 4] batch loss: 1.774\n",
      "[ 6] batch loss: 1.018\n",
      "[ 8] batch loss: 0.858\n",
      "[10] batch loss: 0.477\n",
      "epoch 43 loss: 6.443 -> 10.400\n",
      "LC loss: 0.012 -> 0.012\n",
      "DC loss: 6.431 -> 10.388\n",
      "\n",
      "[ 2] batch loss: 0.546\n",
      "[ 4] batch loss: 0.580\n",
      "[ 6] batch loss: 0.493\n",
      "[ 8] batch loss: 0.510\n",
      "[10] batch loss: 0.497\n",
      "epoch 44 loss: 10.400 -> 5.439\n",
      "LC loss: 0.012 -> 0.012\n",
      "DC loss: 10.388 -> 5.427\n",
      "\n",
      "[ 2] batch loss: 0.565\n",
      "[ 4] batch loss: 0.539\n",
      "[ 6] batch loss: 0.483\n",
      "[ 8] batch loss: 0.502\n",
      "[10] batch loss: 0.452\n",
      "epoch 45 loss: 5.439 -> 5.227\n",
      "LC loss: 0.012 -> 0.011\n",
      "DC loss: 5.427 -> 5.216\n",
      "\n",
      "[ 2] batch loss: 0.379\n",
      "[ 4] batch loss: 0.525\n",
      "[ 6] batch loss: 0.462\n",
      "[ 8] batch loss: 0.473\n",
      "[10] batch loss: 0.421\n",
      "epoch 46 loss: 5.227 -> 4.638\n",
      "LC loss: 0.011 -> 0.011\n",
      "DC loss: 5.216 -> 4.626\n",
      "\n",
      "[ 2] batch loss: 0.339\n",
      "[ 4] batch loss: 0.501\n",
      "[ 6] batch loss: 0.427\n",
      "[ 8] batch loss: 0.446\n",
      "[10] batch loss: 0.399\n",
      "epoch 47 loss: 4.638 -> 4.328\n",
      "LC loss: 0.011 -> 0.011\n",
      "DC loss: 4.626 -> 4.317\n",
      "\n",
      "[ 2] batch loss: 0.311\n",
      "[ 4] batch loss: 0.470\n",
      "[ 6] batch loss: 0.403\n",
      "[ 8] batch loss: 0.420\n",
      "[10] batch loss: 0.372\n",
      "epoch 48 loss: 4.328 -> 4.031\n",
      "LC loss: 0.011 -> 0.011\n",
      "DC loss: 4.317 -> 4.020\n",
      "\n",
      "[ 2] batch loss: 0.287\n",
      "[ 4] batch loss: 0.451\n",
      "[ 6] batch loss: 0.381\n",
      "[ 8] batch loss: 0.396\n",
      "[10] batch loss: 0.348\n",
      "epoch 49 loss: 4.031 -> 3.787\n",
      "LC loss: 0.011 -> 0.011\n",
      "DC loss: 4.020 -> 3.776\n",
      "\n",
      "[ 2] batch loss: 0.267\n",
      "[ 4] batch loss: 0.426\n",
      "[ 6] batch loss: 0.357\n",
      "[ 8] batch loss: 0.373\n",
      "[10] batch loss: 0.330\n",
      "epoch 50 loss: 3.787 -> 3.556\n",
      "LC loss: 0.011 -> 0.010\n",
      "DC loss: 3.776 -> 3.546\n",
      "0.9207547169811321 0.7546359829932638\n",
      "\n",
      "[ 2] batch loss: 0.250\n",
      "[ 4] batch loss: 0.402\n",
      "[ 6] batch loss: 0.335\n",
      "[ 8] batch loss: 0.353\n",
      "[10] batch loss: 0.313\n",
      "epoch 51 loss: 3.556 -> 3.346\n",
      "LC loss: 0.010 -> 0.010\n",
      "DC loss: 3.546 -> 3.335\n",
      "\n",
      "[ 2] batch loss: 0.232\n",
      "[ 4] batch loss: 0.382\n",
      "[ 6] batch loss: 0.313\n",
      "[ 8] batch loss: 0.335\n",
      "[10] batch loss: 0.297\n",
      "epoch 52 loss: 3.346 -> 3.154\n",
      "LC loss: 0.010 -> 0.010\n",
      "DC loss: 3.335 -> 3.144\n",
      "\n",
      "[ 2] batch loss: 0.218\n",
      "[ 4] batch loss: 0.366\n",
      "[ 6] batch loss: 0.294\n",
      "[ 8] batch loss: 0.319\n",
      "[10] batch loss: 0.283\n",
      "epoch 53 loss: 3.154 -> 2.988\n",
      "LC loss: 0.010 -> 0.010\n",
      "DC loss: 3.144 -> 2.978\n",
      "\n",
      "[ 2] batch loss: 0.205\n",
      "[ 4] batch loss: 0.351\n",
      "[ 6] batch loss: 0.278\n",
      "[ 8] batch loss: 0.305\n",
      "[10] batch loss: 0.271\n",
      "epoch 54 loss: 2.988 -> 2.841\n",
      "LC loss: 0.010 -> 0.010\n",
      "DC loss: 2.978 -> 2.831\n",
      "\n",
      "[ 2] batch loss: 0.193\n",
      "[ 4] batch loss: 0.337\n",
      "[ 6] batch loss: 0.263\n",
      "[ 8] batch loss: 0.292\n",
      "[10] batch loss: 0.260\n",
      "epoch 55 loss: 2.841 -> 2.710\n",
      "LC loss: 0.010 -> 0.010\n",
      "DC loss: 2.831 -> 2.700\n",
      "\n",
      "[ 2] batch loss: 0.183\n",
      "[ 4] batch loss: 0.325\n",
      "[ 6] batch loss: 0.250\n",
      "[ 8] batch loss: 0.280\n",
      "[10] batch loss: 0.250\n",
      "epoch 56 loss: 2.710 -> 2.592\n",
      "LC loss: 0.010 -> 0.010\n",
      "DC loss: 2.700 -> 2.582\n",
      "\n",
      "[ 2] batch loss: 0.174\n",
      "[ 4] batch loss: 0.313\n",
      "[ 6] batch loss: 0.238\n",
      "[ 8] batch loss: 0.269\n",
      "[10] batch loss: 0.241\n",
      "epoch 57 loss: 2.592 -> 2.485\n",
      "LC loss: 0.010 -> 0.010\n",
      "DC loss: 2.582 -> 2.475\n",
      "\n",
      "[ 2] batch loss: 0.166\n",
      "[ 4] batch loss: 0.302\n",
      "[ 6] batch loss: 0.228\n",
      "[ 8] batch loss: 0.258\n",
      "[10] batch loss: 0.232\n",
      "epoch 58 loss: 2.485 -> 2.386\n",
      "LC loss: 0.010 -> 0.009\n",
      "DC loss: 2.475 -> 2.376\n",
      "\n",
      "[ 2] batch loss: 0.159\n",
      "[ 4] batch loss: 0.292\n",
      "[ 6] batch loss: 0.218\n",
      "[ 8] batch loss: 0.248\n",
      "[10] batch loss: 0.224\n",
      "epoch 59 loss: 2.386 -> 2.293\n",
      "LC loss: 0.009 -> 0.009\n",
      "DC loss: 2.376 -> 2.284\n",
      "\n",
      "[ 2] batch loss: 0.152\n",
      "[ 4] batch loss: 0.282\n",
      "[ 6] batch loss: 0.209\n",
      "[ 8] batch loss: 0.238\n",
      "[10] batch loss: 0.216\n",
      "epoch 60 loss: 2.293 -> 2.206\n",
      "LC loss: 0.009 -> 0.009\n",
      "DC loss: 2.284 -> 2.197\n",
      "0.9207547169811321 0.8532436394841416\n",
      "\n",
      "[ 2] batch loss: 0.146\n",
      "[ 4] batch loss: 0.273\n",
      "[ 6] batch loss: 0.201\n",
      "[ 8] batch loss: 0.229\n",
      "[10] batch loss: 0.209\n",
      "epoch 61 loss: 2.206 -> 2.125\n",
      "LC loss: 0.009 -> 0.009\n",
      "DC loss: 2.197 -> 2.116\n",
      "\n",
      "[ 2] batch loss: 0.140\n",
      "[ 4] batch loss: 0.265\n",
      "[ 6] batch loss: 0.192\n",
      "[ 8] batch loss: 0.221\n",
      "[10] batch loss: 0.203\n",
      "epoch 62 loss: 2.125 -> 2.050\n",
      "LC loss: 0.009 -> 0.009\n",
      "DC loss: 2.116 -> 2.041\n",
      "\n",
      "[ 2] batch loss: 0.135\n",
      "[ 4] batch loss: 0.257\n",
      "[ 6] batch loss: 0.185\n",
      "[ 8] batch loss: 0.213\n",
      "[10] batch loss: 0.196\n",
      "epoch 63 loss: 2.050 -> 1.978\n",
      "LC loss: 0.009 -> 0.009\n",
      "DC loss: 2.041 -> 1.970\n",
      "\n",
      "[ 2] batch loss: 0.130\n",
      "[ 4] batch loss: 0.250\n",
      "[ 6] batch loss: 0.178\n",
      "[ 8] batch loss: 0.205\n",
      "[10] batch loss: 0.190\n",
      "epoch 64 loss: 1.978 -> 1.912\n",
      "LC loss: 0.009 -> 0.009\n",
      "DC loss: 1.970 -> 1.903\n",
      "\n",
      "[ 2] batch loss: 0.126\n",
      "[ 4] batch loss: 0.243\n",
      "[ 6] batch loss: 0.171\n",
      "[ 8] batch loss: 0.199\n",
      "[10] batch loss: 0.184\n",
      "epoch 65 loss: 1.912 -> 1.850\n",
      "LC loss: 0.009 -> 0.009\n",
      "DC loss: 1.903 -> 1.842\n",
      "\n",
      "[ 2] batch loss: 0.121\n",
      "[ 4] batch loss: 0.236\n",
      "[ 6] batch loss: 0.165\n",
      "[ 8] batch loss: 0.192\n",
      "[10] batch loss: 0.178\n",
      "epoch 66 loss: 1.850 -> 1.791\n",
      "LC loss: 0.009 -> 0.009\n",
      "DC loss: 1.842 -> 1.782\n",
      "\n",
      "[ 2] batch loss: 0.117\n",
      "[ 4] batch loss: 0.229\n",
      "[ 6] batch loss: 0.159\n",
      "[ 8] batch loss: 0.185\n",
      "[10] batch loss: 0.174\n",
      "epoch 67 loss: 1.791 -> 1.733\n",
      "LC loss: 0.009 -> 0.008\n",
      "DC loss: 1.782 -> 1.725\n",
      "\n",
      "[ 2] batch loss: 0.113\n",
      "[ 4] batch loss: 0.225\n",
      "[ 6] batch loss: 0.153\n",
      "[ 8] batch loss: 0.181\n",
      "[10] batch loss: 0.169\n",
      "epoch 68 loss: 1.733 -> 1.689\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 1.725 -> 1.681\n",
      "\n",
      "[ 2] batch loss: 0.114\n",
      "[ 4] batch loss: 0.224\n",
      "[ 6] batch loss: 0.148\n",
      "[ 8] batch loss: 0.182\n",
      "[10] batch loss: 0.164\n",
      "epoch 69 loss: 1.689 -> 1.671\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 1.681 -> 1.662\n",
      "\n",
      "[ 2] batch loss: 0.109\n",
      "[ 4] batch loss: 0.213\n",
      "[ 6] batch loss: 0.154\n",
      "[ 8] batch loss: 0.172\n",
      "[10] batch loss: 0.161\n",
      "epoch 70 loss: 1.671 -> 1.622\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 1.662 -> 1.613\n",
      "0.9220125786163522 0.9307072369557506\n",
      "\n",
      "[ 2] batch loss: 0.115\n",
      "[ 4] batch loss: 0.251\n",
      "[ 6] batch loss: 0.141\n",
      "[ 8] batch loss: 0.191\n",
      "[10] batch loss: 0.179\n",
      "epoch 71 loss: 1.622 -> 1.763\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 1.613 -> 1.755\n",
      "\n",
      "[ 2] batch loss: 0.145\n",
      "[ 4] batch loss: 0.228\n",
      "[ 6] batch loss: 0.194\n",
      "[ 8] batch loss: 0.298\n",
      "[10] batch loss: 0.161\n",
      "epoch 72 loss: 1.763 -> 2.055\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 1.755 -> 2.046\n",
      "\n",
      "[ 2] batch loss: 0.186\n",
      "[ 4] batch loss: 0.466\n",
      "[ 6] batch loss: 0.231\n",
      "[ 8] batch loss: 0.247\n",
      "[10] batch loss: 0.274\n",
      "epoch 73 loss: 2.055 -> 3.082\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 2.046 -> 3.074\n",
      "\n",
      "[ 2] batch loss: 0.302\n",
      "[ 4] batch loss: 0.323\n",
      "[ 6] batch loss: 0.774\n",
      "[ 8] batch loss: 0.807\n",
      "[10] batch loss: 0.443\n",
      "epoch 74 loss: 3.082 -> 6.201\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 3.074 -> 6.193\n",
      "\n",
      "[ 2] batch loss: 0.556\n",
      "[ 4] batch loss: 1.628\n",
      "[ 6] batch loss: 1.537\n",
      "[ 8] batch loss: 1.062\n",
      "[10] batch loss: 0.387\n",
      "epoch 75 loss: 6.201 -> 10.403\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 6.193 -> 10.395\n",
      "\n",
      "[ 2] batch loss: 0.851\n",
      "[ 4] batch loss: 1.115\n",
      "[ 6] batch loss: 0.322\n",
      "[ 8] batch loss: 0.394\n",
      "[10] batch loss: 0.270\n",
      "epoch 76 loss: 10.403 -> 6.178\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 10.395 -> 6.170\n",
      "\n",
      "[ 2] batch loss: 0.288\n",
      "[ 4] batch loss: 0.376\n",
      "[ 6] batch loss: 0.327\n",
      "[ 8] batch loss: 0.353\n",
      "[10] batch loss: 0.275\n",
      "epoch 77 loss: 6.178 -> 3.309\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 6.170 -> 3.302\n",
      "\n",
      "[ 2] batch loss: 0.210\n",
      "[ 4] batch loss: 0.358\n",
      "[ 6] batch loss: 0.282\n",
      "[ 8] batch loss: 0.305\n",
      "[10] batch loss: 0.282\n",
      "epoch 78 loss: 3.309 -> 2.951\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 3.302 -> 2.943\n",
      "\n",
      "[ 2] batch loss: 0.191\n",
      "[ 4] batch loss: 0.335\n",
      "[ 6] batch loss: 0.259\n",
      "[ 8] batch loss: 0.282\n",
      "[10] batch loss: 0.235\n",
      "epoch 79 loss: 2.951 -> 2.633\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 2.943 -> 2.625\n",
      "\n",
      "[ 2] batch loss: 0.169\n",
      "[ 4] batch loss: 0.313\n",
      "[ 6] batch loss: 0.234\n",
      "[ 8] batch loss: 0.259\n",
      "[10] batch loss: 0.218\n",
      "epoch 80 loss: 2.633 -> 2.406\n",
      "LC loss: 0.008 -> 0.008\n",
      "DC loss: 2.625 -> 2.398\n",
      "0.9220125786163522 0.8212825439261191\n",
      "\n",
      "[ 2] batch loss: 0.159\n",
      "[ 4] batch loss: 0.291\n",
      "[ 6] batch loss: 0.213\n",
      "[ 8] batch loss: 0.239\n",
      "[10] batch loss: 0.204\n",
      "epoch 81 loss: 2.406 -> 2.227\n",
      "LC loss: 0.008 -> 0.007\n",
      "DC loss: 2.398 -> 2.219\n",
      "\n",
      "[ 2] batch loss: 0.145\n",
      "[ 4] batch loss: 0.276\n",
      "[ 6] batch loss: 0.198\n",
      "[ 8] batch loss: 0.223\n",
      "[10] batch loss: 0.192\n",
      "epoch 82 loss: 2.227 -> 2.081\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 2.219 -> 2.073\n",
      "\n",
      "[ 2] batch loss: 0.136\n",
      "[ 4] batch loss: 0.263\n",
      "[ 6] batch loss: 0.184\n",
      "[ 8] batch loss: 0.209\n",
      "[10] batch loss: 0.182\n",
      "epoch 83 loss: 2.081 -> 1.956\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 2.073 -> 1.949\n",
      "\n",
      "[ 2] batch loss: 0.127\n",
      "[ 4] batch loss: 0.251\n",
      "[ 6] batch loss: 0.173\n",
      "[ 8] batch loss: 0.196\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10] batch loss: 0.173\n",
      "epoch 84 loss: 1.956 -> 1.850\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.949 -> 1.842\n",
      "\n",
      "[ 2] batch loss: 0.120\n",
      "[ 4] batch loss: 0.241\n",
      "[ 6] batch loss: 0.163\n",
      "[ 8] batch loss: 0.185\n",
      "[10] batch loss: 0.166\n",
      "epoch 85 loss: 1.850 -> 1.755\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.842 -> 1.747\n",
      "\n",
      "[ 2] batch loss: 0.113\n",
      "[ 4] batch loss: 0.231\n",
      "[ 6] batch loss: 0.154\n",
      "[ 8] batch loss: 0.175\n",
      "[10] batch loss: 0.159\n",
      "epoch 86 loss: 1.755 -> 1.670\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.747 -> 1.663\n",
      "\n",
      "[ 2] batch loss: 0.108\n",
      "[ 4] batch loss: 0.222\n",
      "[ 6] batch loss: 0.146\n",
      "[ 8] batch loss: 0.166\n",
      "[10] batch loss: 0.153\n",
      "epoch 87 loss: 1.670 -> 1.595\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.663 -> 1.588\n",
      "\n",
      "[ 2] batch loss: 0.102\n",
      "[ 4] batch loss: 0.214\n",
      "[ 6] batch loss: 0.139\n",
      "[ 8] batch loss: 0.158\n",
      "[10] batch loss: 0.148\n",
      "epoch 88 loss: 1.595 -> 1.527\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.588 -> 1.520\n",
      "\n",
      "[ 2] batch loss: 0.098\n",
      "[ 4] batch loss: 0.207\n",
      "[ 6] batch loss: 0.133\n",
      "[ 8] batch loss: 0.151\n",
      "[10] batch loss: 0.143\n",
      "epoch 89 loss: 1.527 -> 1.467\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.520 -> 1.460\n",
      "\n",
      "[ 2] batch loss: 0.094\n",
      "[ 4] batch loss: 0.201\n",
      "[ 6] batch loss: 0.127\n",
      "[ 8] batch loss: 0.145\n",
      "[10] batch loss: 0.138\n",
      "epoch 90 loss: 1.467 -> 1.414\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.460 -> 1.407\n",
      "0.9220125786163522 0.900783526822456\n",
      "\n",
      "[ 2] batch loss: 0.090\n",
      "[ 4] batch loss: 0.195\n",
      "[ 6] batch loss: 0.122\n",
      "[ 8] batch loss: 0.140\n",
      "[10] batch loss: 0.134\n",
      "epoch 91 loss: 1.414 -> 1.365\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.407 -> 1.358\n",
      "\n",
      "[ 2] batch loss: 0.087\n",
      "[ 4] batch loss: 0.189\n",
      "[ 6] batch loss: 0.117\n",
      "[ 8] batch loss: 0.135\n",
      "[10] batch loss: 0.130\n",
      "epoch 92 loss: 1.365 -> 1.321\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.358 -> 1.314\n",
      "\n",
      "[ 2] batch loss: 0.084\n",
      "[ 4] batch loss: 0.184\n",
      "[ 6] batch loss: 0.113\n",
      "[ 8] batch loss: 0.130\n",
      "[10] batch loss: 0.126\n",
      "epoch 93 loss: 1.321 -> 1.279\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.314 -> 1.272\n",
      "\n",
      "[ 2] batch loss: 0.081\n",
      "[ 4] batch loss: 0.180\n",
      "[ 6] batch loss: 0.109\n",
      "[ 8] batch loss: 0.126\n",
      "[10] batch loss: 0.123\n",
      "epoch 94 loss: 1.279 -> 1.241\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.272 -> 1.234\n",
      "\n",
      "[ 2] batch loss: 0.079\n",
      "[ 4] batch loss: 0.175\n",
      "[ 6] batch loss: 0.105\n",
      "[ 8] batch loss: 0.122\n",
      "[10] batch loss: 0.120\n",
      "epoch 95 loss: 1.241 -> 1.206\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.234 -> 1.199\n",
      "\n",
      "[ 2] batch loss: 0.076\n",
      "[ 4] batch loss: 0.171\n",
      "[ 6] batch loss: 0.102\n",
      "[ 8] batch loss: 0.118\n",
      "[10] batch loss: 0.117\n",
      "epoch 96 loss: 1.206 -> 1.173\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.199 -> 1.166\n",
      "\n",
      "[ 2] batch loss: 0.074\n",
      "[ 4] batch loss: 0.168\n",
      "[ 6] batch loss: 0.099\n",
      "[ 8] batch loss: 0.115\n",
      "[10] batch loss: 0.114\n",
      "epoch 97 loss: 1.173 -> 1.142\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.166 -> 1.135\n",
      "\n",
      "[ 2] batch loss: 0.072\n",
      "[ 4] batch loss: 0.165\n",
      "[ 6] batch loss: 0.096\n",
      "[ 8] batch loss: 0.112\n",
      "[10] batch loss: 0.111\n",
      "epoch 98 loss: 1.142 -> 1.112\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.135 -> 1.106\n",
      "\n",
      "[ 2] batch loss: 0.070\n",
      "[ 4] batch loss: 0.161\n",
      "[ 6] batch loss: 0.093\n",
      "[ 8] batch loss: 0.108\n",
      "[10] batch loss: 0.108\n",
      "epoch 99 loss: 1.112 -> 1.085\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.106 -> 1.078\n",
      "\n",
      "[ 2] batch loss: 0.068\n",
      "[ 4] batch loss: 0.158\n",
      "[ 6] batch loss: 0.090\n",
      "[ 8] batch loss: 0.105\n",
      "[10] batch loss: 0.106\n",
      "epoch 100 loss: 1.085 -> 1.058\n",
      "LC loss: 0.007 -> 0.007\n",
      "DC loss: 1.078 -> 1.052\n",
      "0.9232704402515723 0.9259361578983331\n",
      "\n"
     ]
    }
   ],
   "source": [
    "prev_loss = np.float(\"inf\")\n",
    "prev_loss_LC = np.float(\"inf\")\n",
    "prev_loss_DC = np.float(\"inf\")\n",
    "total_epoch = 100\n",
    "reset_seq(cnn_da.domain_classifier)\n",
    "\n",
    "for epoch in range(total_epoch):\n",
    "    epoch_loss = 0.0\n",
    "    epoch_loss_LC = 0.0\n",
    "    epoch_loss_DC = 0.0\n",
    "    running_loss = 0.0\n",
    "    p = epoch * 1.0 / total_epoch\n",
    "    lr = adjust_lr(optimizer, p)\n",
    "    dslr_iter = iter(trainloader_dslr)\n",
    "    webcam_iter = iter(trainloader_webcam)\n",
    "    i = 0\n",
    "    while True:\n",
    "        try:\n",
    "            images_s, labels_s = dslr_iter.next()\n",
    "            images_t, labels_t = webcam_iter.next()\n",
    "        except:\n",
    "            break\n",
    "        inputs, labels = torch.cat((images_s, images_t)), torch.cat((labels_s, labels_t))\n",
    "        source_size, target_size = labels_s.size(0), labels_t.size(0)\n",
    "        domains = torch.cat((torch.zeros(source_size), torch.ones(target_size)))\n",
    "        if (use_gpu):\n",
    "            inputs, labels, domains = inputs.cuda(), labels.cuda(), domains.cuda()\n",
    "        inputs, labels, domains = Variable(inputs), Variable(labels), Variable(domains)\n",
    "        optimizer.zero_grad()\n",
    "        outputs_LC, outputs_DC = cnn_da(inputs)\n",
    "        loss_LC = criterion_LC(outputs_LC[:source_size], labels[:source_size])\n",
    "        loss_DC = criterion_DC(outputs_DC.view(-1), domains)\n",
    "        loss = loss_LC + loss_DC\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        epoch_loss += loss.item()\n",
    "        epoch_loss_LC += loss_LC.item()\n",
    "        epoch_loss_DC += loss_DC.item()\n",
    "        running_loss += loss.item()\n",
    "        if i % 2 == 1:    # print every 50 mini-batches\n",
    "            print('[%2d] batch loss: %.3f' %\n",
    "                  (i + 1, running_loss / 2))\n",
    "            running_loss = 0.0\n",
    "        i += 1\n",
    "    print(\"epoch %d loss: %.3f -> %.3f\" % (epoch + 1, prev_loss, epoch_loss))\n",
    "    print(\"LC loss: %.3f -> %.3f\" % (prev_loss_LC, epoch_loss_LC))\n",
    "    print(\"DC loss: %.3f -> %.3f\" % (prev_loss_DC, epoch_loss_DC))\n",
    "    if epoch % 10 == 9:\n",
    "        acc_l, acc_d = evaluate_da_accuracy(cnn_da, trainloader_webcam, source=False)\n",
    "        print(acc_l, acc_d)\n",
    "    print()\n",
    "    prev_loss = epoch_loss\n",
    "    prev_loss_LC = epoch_loss_LC\n",
    "    prev_loss_DC = epoch_loss_DC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.save(cnn_da.state_dict(), \"./parameters/cnn_dslr_to_webcam_0.933.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnn_da.load_state_dict(torch.load(\"./parameters/cnn_dslr_to_webcam_0.934.pt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9345911949685535, 0.6322475808221589)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_da_accuracy(cnn_da, trainloader_webcam, source=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}