GAN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RTya_dtvpdaX"
   },
   "source": [
    "# Preparations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "e9W0cTFEPmAl"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import os.path\n",
    "\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import manifold\n",
    "from math import exp\n",
    "from torch.autograd import Variable\n",
    "from my_dataset import MNIST_M\n",
    "from my_dataset import ST_Dataset\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext skip_kernel_extension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this function takes the numpy input image with dim 3 x height x width\n",
    "def convert_to_plt(input_image):\n",
    "    input_image = input_image*np.asarray([0.5,0.5,0.5]) + np.asarray([0.5,0.5,0.5])\n",
    "    return input_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 363,
     "status": "ok",
     "timestamp": 1523506225267,
     "user": {
      "displayName": "Yuan Qi",
      "photoUrl": "//lh6.googleusercontent.com/--bd6SE8_hDo/AAAAAAAAAAI/AAAAAAAAAE0/J27oawL5omk/s50-c-k-no/photo.jpg",
      "userId": "112219197582513023329"
     },
     "user_tz": 420
    },
    "id": "4BrzNF_oQL77",
    "outputId": "fe8e2212-37c8-4027-d176-ef46991bcd01"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "use_gpu = True\n"
     ]
    }
   ],
   "source": [
    "use_gpu = torch.cuda.is_available()\n",
    "print(\"use_gpu = \" + str(use_gpu))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "3W2qqi9azoVs"
   },
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_accuracy(model, dataloader):\n",
    "    model.eval()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for i, data in enumerate(dataloader):\n",
    "        inputs, labels = data\n",
    "        if (use_gpu):\n",
    "            inputs, labels = inputs.cuda(), labels.cuda()\n",
    "        with torch.no_grad():\n",
    "            #inputs, labels = Variable(inputs, volatile=True), Variable(labels, volatile=True)\n",
    "            inputs, labels = Variable(inputs), Variable(labels)\n",
    "            outputs, _, _ = model(inputs)\n",
    "            correct += (torch.max(outputs.data, 1)[1] == labels.data).sum().item()\n",
    "            total += labels.size()[0]\n",
    "    acc = correct * 1.0 / total\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5oBeLwnRLrcY"
   },
   "source": [
    "## Load MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "root_dir = \"./data/mnist/\"\n",
    "\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, ), (0.5, )),\n",
    "    #transforms.Normalize((0.13066047712053577, ), (1, )),\n",
    "    transforms.Lambda(lambda x: torch.cat((x, x, x), dim=0))\n",
    "])\n",
    "\n",
    "trainset = torchvision.datasets.MNIST(root_dir, train=True, transform=transform,\n",
    "                                      target_transform=None, download=True)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,\n",
    "                                          shuffle=False, num_workers=0)\n",
    "testset = torchvision.datasets.MNIST(root_dir, train=False, transform=transform,\n",
    "                                      target_transform=None, download=True)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n",
    "                                          shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 106th image in the first 128 images in the training set:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAFsAAABZCAYAAABR/liSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAA9hJREFUeJztmjssLFEYx39zERSEiMYjtDoh0ehpFAqPRCSEhKm0VCgRBRLxiHiEiKgkRKLTCKVEJAoKCo/Gq9Bg5hZy7skuruXu/cys75dMsZuZOWd/+9+z3zlzHN/3UWT49d0d+EmobEFUtiAqWxCVLYjKFkRlC6KyBVHZgiRLNuY4TsJOV33fdz46R5MtiMoWRGULorIFUdmCiFYjkhQVFQHQ09MDQENDAwDNzc0AbG1tAeB5nlifNNmCOJJPaqTqbMdxWFlZAaC+vv7Nc7KysgC4v7+PS5taZweMhBqz09LSAFhYWHiV6JOTEwDy8/MB6OjoAGB4eFisf5psQRIi2SkpKQDMz88DL+P0zc0NAOPj4wD09fUBsL29DUBysvxH12QLEupkp6amAtDd3Q3YWhpgdnYWgN7e3ohr9vb2AJiYmJDoYgSabEl83xc7AD+eh+u6vuu6vsHzPN/zPH9sbCyu7cRyxPL5NdmChHIGWVhYCMDR0REA6enpAExOTgLQ1dXF09NTPJqKGZ1BBoxQViOmZjaJfnh4AGBgYABAPNWxoskWJFTJrq6uBqClpQV4Wd0DqKmpAeDs7Ox7OhYjmmxBQpXsjIwMAJKSkgC4u7sDbFUSdDTZgoQq2SUlJRGvd3Z2AFt3r6+vv3vt9PR0xDmXl5f/o4t/JVSTms7OTsAuIp2engKQnZ0NQGZm5of3eH5+BuyC1OLiImC/jK+ik5qAEaphJPpRl9muEM3m5uarB7mNjY2A/XOtrKwE7KO03d1dAA4ODuLX4Sg02YKEasxuamoCYGlpKeL9jY0NAPr7+wHY399/tfnGjOe1tbUATE1NAfYBxPn5OQAFBQVf6puO2QEjVMmuq6sDYHV1NeL94uJi4HPT9dvbW8Am3ozxpry8uLj4VN802QEjVNVINGYhKhZMFbK2tgbYqb/BvC4vLwfs/0A80WQLEqpkX11dAfD4+AjYzTmjo6MAuK4bcR5AXl4eYLc2VFVVvXnv5eVl4P8k2qDJFiRU1Yihra0NsFvLTK1sEj0zM0NpaSkAFRUVAOTm5kbcw/w6RkZGABgcHATg+vr6S33SaiRghDLZBjMez83NAVBWVgZATk7Ou9ccHh4CMDQ0BNhVv39Fkx0wQp3saNrb2wFobW39s6pnMOvVZvP78fFxXNvWZAeMhEr2d6LJDhgqWxCVLYjKFkRlC6KyBVHZgqhsQVS2IKIzyJ+OJlsQlS2IyhZEZQuisgVR2YKobEFUtiAqWxCVLYjKFkRlC6KyBVHZgqhsQVS2ICpbEJUtiMoWRGULorIFUdmCqGxBfgOTPcpvgiM1eAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 72x72 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Its correspondent label:\n",
      "tensor(6)\n"
     ]
    }
   ],
   "source": [
    "# randomly plot a sample from training set\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "inputs = None\n",
    "labels = None\n",
    "for i, data in enumerate(trainloader):\n",
    "    inputs, labels = data\n",
    "    break\n",
    "\n",
    "idx = np.random.randint(0, batch_size)\n",
    "print(\"The \" + str(idx) + \"th image in the first \" + str(batch_size) +\\\n",
    "      \" images in the training set:\")\n",
    "\n",
    "plt_img = convert_to_plt(inputs[idx].permute(1, 2, 0).cpu().data.numpy())\n",
    "plt.figure(figsize=(1,1))\n",
    "plt.axis(\"off\")\n",
    "plt.imshow(plt_img)\n",
    "plt.show()\n",
    "print(\"Its correspondent label:\\n\" + str(labels[idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST-M Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_dataset import MNIST_M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "root_dir = \"./data/\"\n",
    "\n",
    "transform_m = transforms.Compose([\n",
    "    transforms.Resize((28, 28)),\n",
    "    transforms.ToTensor(),\n",
    "    #transforms.Normalize((0.4581609321206303, 0.462350402961343, 0.4084781187671726), (1, 1, 1))\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "])\n",
    "\n",
    "trainset_m = MNIST_M(root_dir, train=True, transform=transform_m, download=True)\n",
    "trainloader_m = torch.utils.data.DataLoader(trainset_m, batch_size=batch_size,\n",
    "                                          shuffle=False, num_workers=0)\n",
    "testset_m = MNIST_M(root_dir, train=False, transform=transform_m, download=True)\n",
    "testloader_m = torch.utils.data.DataLoader(testset_m, batch_size=batch_size,\n",
    "                                          shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 0th image in the first 128 images in the training set:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAFsAAABZCAYAAABR/liSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAC1JJREFUeJztXEuPXEcZPffZ3bcf0z0948nEIbYxiYLCAokVgh8AKPwQ/gY7fgFCrGDFBgGREAv4CSxQ4kQiKA6JM/bY8/DM9OM++j5YnPPZsjdpE6kYifo2re6uW7du9alT53tUB13XwZsbC//XA/h/Mj/ZDs1PtkPzk+3Q/GQ7ND/ZDs1PtkPzk+3Q/GQ7tNjlzX71i593AFBtKgDA4vQEAHD18BgAsBNxOEMNq8oLAMC6aQAAe7e/AQAYZEMAwAf37gEA9vV5HrYAgLKu8Pbb7wAATp+c6e4BAGB3dx8AUHd8Px6PeW21AQAcnzxhH0UOAJiOJwCAOGD7w/keAGA06AMA/vT73wEAfv3nvwZf9fwe2Q7NKbLjKAIAFBXjMXHM94k+bzdEcBMRA0XL90HWAwCUIHInfb6PArarixIAsKjWAIDBZIQg5D3CiK9NzWuvri40Gt5zsVgAAPYObgAAsh77LpbsK2h5vYCN1ZLtb8x2AACjQbb183tkOzSnyG5aokt0CYs3BoJNkqb6gKgrQzUU8i/W5NG6Jq9W4tlQfGzISdMYre6VCXmLqyW71mrZ3ZsDAAbZiLdIieizk1OOTe2ikL12jfaDivtNWRZ6pmbr5/fIdmhOkR0KoakQHEtV1AlRFQrhYczvI8XawyHR2Qrp5xdXAIBOS6Spia5EaibrD5Em7KMC+RxCYKJ9YpQNAACHr7/GvoIEAHD04AgAMJ9OAQAH++TyJ8dUTLZPVEK4raqtnn/rlt6+tjlF9jon58ax1IY4uKzJvUHI4QQp0deTxh3ucOdHyM+PF7yuqmsAwPkFFUbTY7/7QYSyYJ+x0J6Kk0NxcGWcu2G7tEdk704mL7SLheSdMbl9Z0JdPhiwvyzzauRamlNkF0K2QIONuLaWlu1SoqtL+JqvyLeTgPybxIn60UoQ91+tVgCAqOP7vCgQ5uTUs46eXpDRc2zkjX54/xEA4CdzcvKtERE9FVLTHq+THEcam8dJhGd93sv2oW3MI9uhOUV2JBRYnAFCaiAuRmKcTdT0GnmI8iwrobXXp5KIe2z/5aMHAIBJTHVT1R3ygMisY3KrZDIqraJFwzHkJVfJ4ooKp8zlOdqgBcdCsZKmZfslm2OyM9n6+T2yHZpbZEvjToZE4Kqimmgbom2leARMZ5seF5dfLMjNN24cAAAen5B3s50ZX3fpFXb7h7has20i6g2lu0dSG5VUyC//8jcAzzn5Zz/4IQCgr9WTaPWscu0Lits0ilyu1+utn98j26E5RTaEitbUhxAddoYWEuumoAaOJ9rxha7LnBG38YQrI9D13/vxewCA/mtE/OnRFwgikupFTkUTaL+Yz+gZWiFYoYhh1+P3G5F7pH2k35OOFok/fszVFCr20jY+NnItzSmyLV5hsLKYRrcxpBMtFkuOpXkfHDNeUXbk+EZx7Z4id6P9XQBAG7CffpIinNDrXBTM1ATClcWf012+v/3GmwCAp5eM9vWVgVkZF0cvhigv5K2apznoew/yWppbNWJxZ2neVhwd6PNWkTTDQCiOXy/J4UPlC1MheTynV5gdvg4AaFaMWZejMYrLSwDPVcVASK8viODloy8BAPMx+X+5Isd/rOjerRE/jy1O0+OqnI7Zz0Rxm88Wn279/B7ZDs1tbGRFHgzl1ZlGtQhcLS9tnFExJDGHZxmeQsT5ZcuVMOqTX/OSK8IyPkUIRGdPAQB7HfeFi39/AgBYdNTXT4/odX7RcCyPpVr+/vkXAIB2xM/j9C0AQKYoX08xk1j7TyHltI15ZDs0p8i2HT4e8Ladfuu6pcqA9HdPMejIkLrh97M7dwAApxXR2hP312pnuc3zssBU9R+ifawfEslz8X2oe10pErlcE6GWY6yfxWX4fqBs+lBcnorDw9Bnaq6ludXZqslYi6NbBfuChL95kCoSl1NV9Crz0ojCZIdqZKz3sTTxh++/z+ulJHYHI9QlEXtRWP0HuToT55oyihMNQh5jo76HQ6qYy0ty/1A5y+mUcRjzFaJXiGc7neyeHIrHZ3R5lyqYsaXeU2qqsw1xxR+lP2TAfrmmu95mnPTq3j/4+gFfA7n96X7wLEGRyClplRZrtcm+vPhrud07knSHh0wE337zFsfyzO3nlJW5lTJs//yeRhyaU2RnAyLUytACIdgQ3VMiOJU0HCnRe/tHDHvev/8ZAGBpvrOVJ+g1lFTcdDWyPjeyUKGAIOK9GtFIeniT157Q+ZmDY3jr7rcAAIMBx2LOS1tzFa5WSlJL8oXB9nj1yHZojgsriR4rD0gl2WJtNrUC+tlArrJtPiLYkZIO5SnROJCTtFGCuBFRN+hQqswhEto7pdzO+9zgBnfJxdN//RMAMBZiZwnHMtWq6ve4MaKjO2+OmI3VS79ram6TBw3RlgmJa1Fvrp29WlFtjCZEX1VSAUiNPS9fU5gzeEhVE6k/k281AvSVWosl4SoVsecqi8i0Ct6dMzSwURfB6UMAwPTtb/J6rUYzK95p8QoyxK595Su8/dfmtmR4Q6SmQkeocrNcSYSiVhJBWtZCrI0wsZaoDcTDtdx4XQYJDyRRjEgh1ad3qDqqSEXuT7gqLj/5GADw/UMivpur/YIO1UZ9W7G8OS8HB0y9nZw8BoBnpcnbmEe2Q3OKbAulWmlwo528Sal97SjG0orcVTCTiotPVYg5fJMHloJ9Jg/6QvxcRT/laoEjqYZWyeF0o9VhhZVK3CY3uD9YCu6RDjxZ8byFCjYKSO1MqbtHI/oMr/IHIh7ZDs0psodDIrhSiHQjHjS0lfrtF4XKzFIdqVDsdClJ28jjTJTS2hGiM+0Bi09XqNXnyI7gdezrbqYjeD15liocSq30V/tFrjFMxP3G4UsdYDLv1ZcyXFNziuzAQhoqK9tEKjMbEplhQ5Qtleaa1FQv9ZKlX3s3ydWRAvdr8WonQZAK4RcnFwhLcnbxEVXHYMZIYXLzQG356IlKf63W0yKOp6fkbkPuUEdNAvN+lRZ7lRCrR7ZDc4rso0dUAOM5Y8VNSrQkSipsjMuXOnSkvf7Bb34LANh976cAgFpJ3FQroFGKa6l4+SzLsN8nQu8fkaMbcS42lktjH7kUj6XDQi0Ti4EU8m4PDqh8kphqxI57zGazrZ/fI9uhuT2aJ71s5bsrJVsLadiuVOzEInXix0DRveAPf+T30ucTIXkpyHx0eQ4AeOe77yIYMCYSSm1Ysnmjg01WUFk9S/ASwaan92c8/nF2yqKe0xNy+EAcb1HBsfT2Vs+/dUtvX9ucIvtSRynQJ+9Z0nWgWHNb83V9RvXRKGJnBTGVYivBS0ogVIbHjmPXdfPsKJ4laK/Ombi1qJ2VvK3klU5nLHGIYuroiTjZinBqrYBcq/HkhH/f4Y9TX1NziuylImhxRuSaIohC/dXFSxkbO5KxUMFkLWSHaleJd6EVkAjNXQcspc1bK0+24vXWguMvFuZPxMFzHT05Ozt74fu+9gBbMSsdBywqU05fbR7ZDs0psp+qjLcOWRoW63B+Is4t5WLuzhWJ0x8I5OJLZQMRS5d3ypQ3imcYaPN1jljZmzR5MdOSC4mlVk+ubNDa/iBGhfiff85ytUh97+/tvTC24YicPlS+dBvzyHZobrPrUh22w2cZkfnGIeMVDx8QXRsVWl4pa1IoU54pcxMoBgKpj5X6u9SeMNvfRaa/12girgrj7rNzcvFa11i0byEdfvyEq+5E7SwbZNxsZWnf+Tb/8KunsuVtzCPboQX+z8rdmUe2Q/OT7dD8ZDs0P9kOzU+2Q/OT7dD8ZDs0P9kOzU+2Q/OT7dD8ZDs0P9kOzU+2Q/OT7dD8ZDs0P9kOzU+2Q/OT7dD8ZDs0P9kOzU+2Q/OT7dD8ZDu0/wDGETM6hWhaHgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 72x72 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Its correspondent label:\n",
      "tensor(0)\n"
     ]
    }
   ],
   "source": [
    "# randomly plot a sample from training set\n",
    "\n",
    "inputs = None\n",
    "labels = None\n",
    "for i, data in enumerate(trainloader_m):\n",
    "    inputs, labels = data\n",
    "    break\n",
    "    \n",
    "idx = np.random.randint(0, batch_size)\n",
    "print(\"The \" + str(idx) + \"th image in the first \" + str(batch_size) +\\\n",
    "      \" images in the training set:\")\n",
    "\n",
    "plt_img = convert_to_plt(inputs[idx].permute(1, 2, 0).cpu().data.numpy())\n",
    "plt.figure(figsize=(1,1))\n",
    "plt.axis(\"off\")\n",
    "plt.imshow(plt_img)\n",
    "plt.show()\n",
    "print(\"Its correspondent label:\\n\" + str(labels[idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import manifold\n",
    "\n",
    "def extract_features(model, dataloader):\n",
    "    features = None\n",
    "    lasthidden = None\n",
    "    labels = None\n",
    "    for i, data in enumerate(dataloader):\n",
    "        inputs, labels_ = data\n",
    "        if (use_gpu):\n",
    "            inputs = inputs.cuda()\n",
    "        inputs = Variable(inputs)\n",
    "        _, f, l = model(inputs)\n",
    "        if i == 0:\n",
    "            features = f\n",
    "            lasthidden = l\n",
    "            labels = labels_\n",
    "        else:\n",
    "            features = torch.cat((features, f))\n",
    "            lasthidden = torch.cat((lasthidden, l))\n",
    "            labels = torch.cat((labels, labels_))\n",
    "    features = features.data\n",
    "    lasthidden = lasthidden.data\n",
    "    if use_gpu:\n",
    "        features = features.cpu()\n",
    "        lasthidden = lasthidden.cpu()\n",
    "    return features, lasthidden, labels\n",
    "\n",
    "def visualize_single_dataset(data, labels, perplexity=50, sample_num=None):\n",
    "    total_num = labels.shape[0]\n",
    "    if sample_num:\n",
    "        idx = np.random.choice(total_num, sample_num, replace=False)\n",
    "        data, labels = data[idx, :], labels[idx]\n",
    "        total_num = sample_num\n",
    "    tsne = manifold.TSNE(n_components=2, init='random',\n",
    "                     random_state=0, perplexity=perplexity)\n",
    "    X = tsne.fit_transform(data)\n",
    "    colors = [\"red\", \"orange\", \"goldenrod\", \"yellow\", \"yellowgreen\", \"green\", \"teal\", \"blue\", \"violet\", \"purple\"]\n",
    "    for i in range(10):\n",
    "        plt.scatter(X[labels == i, 0], X[labels == i, 1], c=colors[i], alpha=0.4)\n",
    "    plt.axis(\"off\")\n",
    "    \n",
    "def visualize_da(source, target, perplexity=50, sample_num=None, save=None):\n",
    "    source_num = source.shape[0]\n",
    "    target_num = target.shape[0]\n",
    "    if sample_num:\n",
    "        source, target = source[:sample_num, :], target[:sample_num, :]\n",
    "        \n",
    "    data = np.vstack((source, target))\n",
    "\n",
    "    tsne = manifold.TSNE(n_components=2, init='random',\n",
    "                         random_state=0, perplexity=perplexity)\n",
    "    X = tsne.fit_transform(data)\n",
    "    plt.scatter(X[:sample_num, 0], X[:sample_num, 1], c=\"blue\", edgecolors=None, alpha=0.4)\n",
    "    plt.scatter(X[sample_num:, 0], X[sample_num:, 1], c=\"red\", edgecolors=None, alpha=0.4)\n",
    "    plt.axis(\"off\")\n",
    "    if save:\n",
    "        plt.savefig(save)\n",
    "    plt.show()\n",
    "    plt.clf()\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_dataset import ST_Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "\n",
    "trainset_da = ST_Dataset(trainset, trainset_m, batch_size)\n",
    "trainloader_da = torch.utils.data.DataLoader(trainset_da, batch_size=batch_size,\n",
    "                                          shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 22th image in the first 128 images in the training set:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAFsAAABZCAYAAABR/liSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAA5RJREFUeJzt278rfXEcx/Hn4RvKr/KjlPIzC0oGA4MSmQw2g1IGGWShyGCxsJqUhcUgJeUPsNhMRCFJYrihUH6W7ne4nXvcL/frxve+7rnn+34s6jrOeffq1elz7udwwuEwRiMj1QP8TyxsIQtbyMIWsrCFLGwhC1vIwhaysIV+KS/mOE5gH1fD4bDz1THWbCELW8jCFrKwhSxsIQtbyMIWsrCFLGwh6RNkKhQVFQGws7MDQHV1NQB5eXkAPD4+ymaxZgsFvtlTU1MAVFVVAXB5eQnA29ubfBZrtlBgmz07OwvA+Pg4AO77Mf39/QC8vLzIZ7JmCwWq2ZmZmUCk1WNjY58ec3NzoxwphjVbKBDNzsiIdObP+7TfWLOF0rrZFRUVANH78+joaNxjQ6EQAHd3d8kfLA5rtlBaNru3txeAubk5AOrq6gB4fX2Nfj44OAh47T84OADg4uJCOut71myhtGr20NAQABMTEwDU1NQAXqObmpoAOD4+ZmBgIOZvU7m+dlmzhXzd7JKSEgDa2toAmJ+fByA7OxuAra0tAGZmZoBIowFaW1spKyuLOdfKykryB/6CL8MuKCgAYGNjA4iE997y8jIAk5OTAFxfX8f8vru7m5ycnB/N0NXVBUBDQwMAS0tLANzf33/7nHYbEfJlszc3NwHv9uF6enoCYG1tDYD8/PyYn1dXVwC0t7fjOJH3HN3Ngv39fcDbFnO5t5u+vj4AWlpaYq798PAAwOnpacxs32HNFnKU/3Sa6CvDIyMjANEHk+bm5oTOv7e3B0B5eTnFxcXuNQFv8yBRh4eHAExPTwOwvr7+1+PtlWGf8WWzXbm5uQD09PQA0NnZCUBHRwfgPdR8cU3gY7Ofn5+Bj0vCxcVFAE5OTgC4vb1NaFZrts/4utnxZGVlAd422PDwMOC9rvD+q9bz83MA6uvrPz2Xu8L5KWu2z6Rls+MpLCwEYHt7O9rks7MzAGpra5N5aWu23/jyCfK73C2vUCgUbfbq6moqR4phzRYKVLNLS0sBqKysjH72k+8y/jVrtlCgViONjY0A7O7uRj9z1+LJZqsRn7GwhSxsoUCtRo6OjgBYWFj4sG/pB9ZsoUCtRlLJViM+Y2ELWdhCFraQhS0kXY3876zZQha2kIUtZGELWdhCFraQhS1kYQtZ2EIWtpCFLWRhC1nYQha2kIUtZGELWdhCFraQhS1kYQtZ2EIWtpCFLfQbbgvx6teFzQMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 72x72 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Its correspondent label:\n",
      "4\n",
      "\n",
      "From domain:\n",
      "Source\n"
     ]
    }
   ],
   "source": [
    "# randomly plot a sample from test set\n",
    "\n",
    "inputs = None\n",
    "labels = None\n",
    "for i, data in enumerate(trainloader_da):\n",
    "    inputs, labels, domain = data\n",
    "    if i == 2:\n",
    "        break\n",
    "\n",
    "idx = np.random.randint(0, batch_size)\n",
    "print(\"The \" + str(idx) + \"th image in the first \" + str(batch_size) +\\\n",
    "      \" images in the training set:\")\n",
    "plt_img = convert_to_plt(inputs[idx].permute(1, 2, 0).cpu().data.numpy())\n",
    "plt.figure(figsize=(1,1))\n",
    "plt.axis(\"off\")\n",
    "plt.imshow(plt_img)\n",
    "plt.show()\n",
    "print(\"Its correspondent label:\")\n",
    "if labels[idx].item() == -1:\n",
    "    print(\"I don't know :)\\n\")\n",
    "else:\n",
    "    print(str(labels[idx].item()) + \"\\n\")\n",
    "\n",
    "print(\"From domain:\")\n",
    "if domain[idx].item() == 0:\n",
    "    print(\"Source\")\n",
    "else:\n",
    "    print(\"Target\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GAN Algorithm\n",
    "## Definition\n",
    "Define label classifier $L$ has parameter $\\theta_{lc}$, domain classifier $D$ has parameter $\\theta_{dc}$ and feature extractor $F$ has parameter $\\theta_{fe}$  \n",
    "$x_s$ is the data from source domain, while $x_t$ is the data from target domain  \n",
    "$y_s$ is the label for source data  \n",
    "\n",
    "## Loss\n",
    "Loss function $L= log(D(F(x_t)))+log(1-D(F(x_s)))+L(F(x_s),y_s)$  \n",
    "$D(F(x_t))$ will output 1 if it thinks input is from target, else output 0  \n",
    "\n",
    "## Update  \n",
    "$\\theta_{lc}$  minimize  $L(F(x_s),y_s)$  \n",
    "$\\theta_{dc}$  minimize  $-log(D(F(x_t)))-log(1-D(F(x_s)))$  \n",
    "$\\theta_{fe}$  minimize  $log(D(F(x_t)))+log(1-D(F(x_s)))+L(F(x_s),y_s)$  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super(CNN, self).__init__()\n",
    "        # feature extractor\n",
    "        self.C1 = nn.Conv2d(3, 32, 5)\n",
    "        self.C2 = nn.Conv2d(32, 48, 5)\n",
    "      \n",
    "        self.C1.weight.data.normal_(0.0, 0.1)\n",
    "        self.C2.weight.data.normal_(0.0, 0.1)    \n",
    "        \n",
    "        # label classifier\n",
    "        self.LC_FC1 = nn.Linear(48 * 8 * 8, 100)\n",
    "        self.LC_FC2 = nn.Linear(100, 100)\n",
    "        self.LC_FC3 = nn.Linear(100, 10)\n",
    "        \n",
    "        \n",
    "        self.LC_FC1.weight.data.normal_(0.0, 0.1)\n",
    "        self.LC_FC2.weight.data.normal_(0.0, 0.1)\n",
    "        self.LC_FC3.weight.data.normal_(0.0, 0.1)\n",
    "        '''\n",
    "        self.LC_FC1.bias.data.normal_(0.0, 0.1)\n",
    "        self.LC_FC2.bias.data.normal_(0.0, 0.1)\n",
    "        self.LC_FC3.bias.data.normal_(0.0, 0.1)\n",
    "        '''\n",
    "        # domain classifier\n",
    "        self.DC_FC1 = nn.Linear(48 * 8 * 8, 100)\n",
    "        self.DC_FC2 = nn.Linear(100, 100)\n",
    "        self.DC_FC3 = nn.Linear(100, 10)\n",
    "        self.DC_FC4 = nn.Linear(10, 1)\n",
    "        \n",
    "        \n",
    "        self.DC_FC1.weight.data.normal_(0.0, 0.1)\n",
    "        self.DC_FC2.weight.data.normal_(0.0, 0.1)\n",
    "        self.DC_FC3.weight.data.normal_(0.0, 0.1)\n",
    "        self.DC_FC4.weight.data.normal_(0.0, 0.1)\n",
    "        '''\n",
    "        self.DC_FC1.bias.data.normal_(0.0, 0.1)\n",
    "        self.DC_FC2.bias.data.normal_(0.0, 0.1)\n",
    "        self.DC_FC3.bias.data.normal_(0.0, 0.1)\n",
    "        self.DC_FC4.bias.data.normal_(0.0, 0.1) \n",
    "        '''\n",
    "        \n",
    "    def forward(self, x):\n",
    "        # C1\n",
    "        x = F.relu(self.C1(x))\n",
    "        # M1\n",
    "        x = F.max_pool2d(x, (2, 2))\n",
    "        # C2\n",
    "        x = F.relu(self.C2(x))\n",
    "        \n",
    "        x = x.view(-1 , 48 * 8 * 8)\n",
    "        \n",
    "        f = x\n",
    "        # label classifier\n",
    "        # LC_FC1\n",
    "        x_l = F.relu(self.LC_FC1(f))\n",
    "        # LC_FC2\n",
    "        x_l = F.relu(self.LC_FC2(x_l))\n",
    "        # LC_FC3\n",
    "        x_l = self.LC_FC3(x_l)\n",
    "\n",
    "        \n",
    "        # discriminator classifier\n",
    "        # DC_FC1\n",
    "        x_d = F.elu(self.DC_FC1(f))\n",
    "        # DC_FC2\n",
    "        x_d = F.elu(self.DC_FC2(x_d))\n",
    "        # DC_FC3\n",
    "        x_d = F.elu(self.DC_FC3(x_d))\n",
    "        # DC_FC4\n",
    "        x_d = F.sigmoid(self.DC_FC4(x_d))\n",
    "        \n",
    "        \n",
    "        eps = 1e-5\n",
    "        x_d = torch.clamp(x_d, min=eps, max=1-eps)\n",
    "        \n",
    "        return x_l, x_d\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_features_da(model, dataloader):\n",
    "    features = None\n",
    "    lasthidden = None\n",
    "    labels = None\n",
    "    for i, data in enumerate(dataloader):\n",
    "        inputs, labels_ = data\n",
    "        if (use_gpu):\n",
    "            inputs = inputs.cuda()\n",
    "        inputs = Variable(inputs)\n",
    "        _, _, f, l = model(inputs)\n",
    "        \n",
    "        # from source\n",
    "        if i == 0:\n",
    "            features = f\n",
    "            lasthidden = l\n",
    "            labels = labels_\n",
    "        # from target\n",
    "        else:\n",
    "            features = torch.cat((features, f))\n",
    "            lasthidden = torch.cat((lasthidden, l))\n",
    "            labels = torch.cat((labels, labels_))\n",
    "    features = features.data\n",
    "    lasthidden = lasthidden.data\n",
    "    if use_gpu:\n",
    "        features = features.cpu()\n",
    "        lasthidden = lasthidden.cpu()\n",
    "    return features, lasthidden, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN(\n",
      "  (C1): Conv2d(3, 32, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (C2): Conv2d(32, 48, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (LC_FC1): Linear(in_features=3072, out_features=100, bias=True)\n",
      "  (LC_FC2): Linear(in_features=100, out_features=100, bias=True)\n",
      "  (LC_FC3): Linear(in_features=100, out_features=10, bias=True)\n",
      "  (DC_FC1): Linear(in_features=3072, out_features=100, bias=True)\n",
      "  (DC_FC2): Linear(in_features=100, out_features=100, bias=True)\n",
      "  (DC_FC3): Linear(in_features=100, out_features=10, bias=True)\n",
      "  (DC_FC4): Linear(in_features=10, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net = CNN()\n",
    "if (use_gpu):\n",
    "    net.cuda()\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import exp\n",
    "\n",
    "criterion_LC = nn.CrossEntropyLoss()\n",
    "\n",
    "params_FE =[net._modules['C1'].weight,net._modules['C1'].bias, \\\n",
    "            net._modules['C2'].weight,net._modules['C2'].bias]\n",
    "\n",
    "params_LC =[net._modules['LC_FC1'].weight,net._modules['LC_FC1'].bias, \\\n",
    "            net._modules['LC_FC2'].weight,net._modules['LC_FC2'].bias, \\\n",
    "            net._modules['LC_FC3'].weight,net._modules['LC_FC3'].bias]\n",
    "\n",
    "\n",
    "criterion_DC = nn.BCELoss()\n",
    "params_DC =[net._modules['DC_FC1'].weight,net._modules['DC_FC1'].bias, \\\n",
    "            net._modules['DC_FC2'].weight,net._modules['DC_FC2'].bias, \\\n",
    "            net._modules['DC_FC3'].weight,net._modules['DC_FC3'].bias, \\\n",
    "            net._modules['DC_FC4'].weight,net._modules['DC_FC4'].bias]\n",
    "\n",
    "\n",
    "\n",
    "def adjust_lr(optimizer, p, lr_init):\n",
    "    lr_0 = lr_init\n",
    "    alpha = 10\n",
    "    beta = 0.75\n",
    "    lr = lr_0 / (1 + alpha * p) ** beta\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "    return lr\n",
    "        \n",
    "def adjust_lamda(model, p):\n",
    "    gamma = 10\n",
    "    lamda = 2 / (1 + exp(- gamma * p)) - 1\n",
    "    model.set_lamda(lamda)\n",
    "    return lamda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load_model_da = True\n"
     ]
    }
   ],
   "source": [
    "para_file_da = \"./parameters/DA_GAN.pt\"\n",
    "\n",
    "load_model_da = os.path.isfile(para_file_da)\n",
    "\n",
    "print(\"load_model_da = \" + str(load_model_da))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%skip (not $para_file_da)\n",
    "#net.load_state_dict(torch.load(para_file_da))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_da_accuracy(model ,dataloader, source):\n",
    "    model.eval()\n",
    "    correct_LC = 0\n",
    "    total = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for i, data in enumerate(dataloader):\n",
    "            inputs, labels = data\n",
    "            if (use_gpu):\n",
    "                inputs, labels = inputs.cuda(), labels.cuda()\n",
    "            inputs, labels = Variable(inputs, volatile = True), Variable(labels, volatile = True)\n",
    "\n",
    "            \n",
    "            outputs_LC, _ = model(inputs)\n",
    "            correct_LC += (torch.max(outputs_LC.data, 1)[1] == labels.data).sum().item()\n",
    "\n",
    "            total += labels.size()[0]\n",
    "        acc_LC = correct_LC / total\n",
    "    return acc_LC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------\n",
      "Start domain classifier training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py:1474: UserWarning: Using a target size (torch.Size([64])) that is different to the input size (torch.Size([64, 1])) is deprecated. Please ensure they have the same size.\n",
      "  \"Please ensure they have the same size.\".format(target.size(), input.size()))\n",
      "/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py:1474: UserWarning: Using a target size (torch.Size([56])) that is different to the input size (torch.Size([56, 1])) is deprecated. Please ensure they have the same size.\n",
      "  \"Please ensure they have the same size.\".format(target.size(), input.size()))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 DC iter 1 acc 0.852 loss: inf -> 235.588\n",
      "\n",
      "Epoch 0 DC iter 2 acc 0.852 loss: 235.588 -> 442.852\n",
      "\n",
      "Epoch 0 DC iter 3 acc 0.875 loss: 235.588 -> 59.582\n",
      "\n",
      "Epoch 0 DC iter 4 acc 0.875 loss: 59.582 -> 29.770\n",
      "\n",
      "Epoch 0 DC iter 5 acc 0.875 loss: 29.770 -> 23.130\n",
      "\n",
      "Epoch 0 DC iter 6 acc 0.875 loss: 23.130 -> 14.423\n",
      "\n",
      "Epoch 0 DC iter 7 acc 0.867 loss: 14.423 -> 57.046\n",
      "\n",
      "Epoch 0 DC iter 8 acc 0.852 loss: 14.423 -> 7.018\n",
      "\n",
      "Epoch 0 DC iter 9 acc 0.859 loss: 7.018 -> 10.561\n",
      "\n",
      "Epoch 0 DC iter 10 acc 0.867 loss: 7.018 -> 9.137\n",
      "\n",
      "Epoch 0 DC iter 11 acc 0.875 loss: 7.018 -> 5.958\n",
      "\n",
      "Epoch 0 DC iter 12 acc 0.875 loss: 5.958 -> 23.365\n",
      "\n",
      "Epoch 0 DC iter 13 acc 0.875 loss: 5.958 -> 6.278\n",
      "\n",
      "Epoch 0 DC iter 14 acc 0.875 loss: 5.958 -> 30.586\n",
      "\n",
      "Epoch 0 DC iter 15 acc 0.852 loss: 5.958 -> 3.353\n",
      "\n",
      "Epoch 0 DC iter 16 acc 0.875 loss: 3.353 -> 11.189\n",
      "\n",
      "Epoch 0 DC iter 17 acc 0.875 loss: 3.353 -> 6.223\n",
      "\n",
      "Epoch 0 DC iter 18 acc 0.875 loss: 3.353 -> 5.364\n",
      "\n",
      "Epoch 0 DC iter 19 acc 0.875 loss: 3.353 -> 4.291\n",
      "\n",
      "Epoch 0 DC iter 20 acc 0.875 loss: 3.353 -> 9.935\n",
      "\n",
      "-----------------------\n",
      "Start label classifier training\n",
      "Epoch 0 LC iter 1 LC_acc 1.000 loss: inf -> 249.722\n",
      "\n",
      "Epoch 0 LC iter 2 LC_acc 1.000 loss: 249.722 -> 74.812\n",
      "\n",
      "Epoch 0 LC iter 3 LC_acc 1.000 loss: 74.812 -> 47.369\n",
      "\n",
      "Epoch 0 LC iter 4 LC_acc 1.000 loss: 47.369 -> 34.017\n",
      "\n",
      "Epoch 0 LC iter 5 LC_acc 1.000 loss: 34.017 -> 25.536\n",
      "\n",
      "Epoch 0 LC iter 6 LC_acc 1.000 loss: 25.536 -> 19.457\n",
      "\n",
      "Epoch 0 LC iter 7 LC_acc 1.000 loss: 19.457 -> 15.097\n",
      "\n",
      "Epoch 0 LC iter 8 LC_acc 1.000 loss: 15.097 -> 11.865\n",
      "\n",
      "Epoch 0 LC iter 9 LC_acc 1.000 loss: 11.865 -> 9.321\n",
      "\n",
      "Epoch 0 LC iter 10 LC_acc 1.000 loss: 9.321 -> 7.336\n",
      "\n",
      "-----------------------\n",
      "Start feature extractor training\n",
      "Epoch 0 FE iter 1 LC_acc 1.000 DC_acc 0.438 loss: inf -> 797.530\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:11: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label classifier accuracy on MNIST-M test set (DA): 0.409667\n",
      "\n",
      "Epoch 0 FE iter 2 LC_acc 1.000 DC_acc 0.438 loss: 797.530 -> 91.017\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.428333\n",
      "\n",
      "Epoch 0 FE iter 3 LC_acc 1.000 DC_acc 0.438 loss: 91.017 -> 56.829\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.409111\n",
      "\n",
      "Epoch 0 FE iter 4 LC_acc 1.000 DC_acc 0.438 loss: 56.829 -> 48.414\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.403333\n",
      "\n",
      "Epoch 0 FE iter 5 LC_acc 1.000 DC_acc 0.438 loss: 48.414 -> 37.626\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.406111\n",
      "\n",
      "Epoch 0 FE iter 6 LC_acc 1.000 DC_acc 0.438 loss: 37.626 -> 33.749\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.403667\n",
      "\n",
      "Epoch 0 FE iter 7 LC_acc 1.000 DC_acc 0.438 loss: 33.749 -> 35.496\n",
      "\n",
      "-----------------------\n",
      "Start domain classifier training\n",
      "Epoch 1 DC iter 1 acc 0.438 loss: inf -> 1445.302\n",
      "\n",
      "Epoch 1 DC iter 2 acc 0.438 loss: 1445.302 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 3 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 4 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 5 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 6 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 7 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 8 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 9 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 10 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 11 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 12 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 13 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 14 acc 0.438 loss: 1278.163 -> 1278.216\n",
      "\n",
      "Epoch 1 DC iter 15 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 16 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 17 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 18 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 19 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 1 DC iter 20 acc 0.438 loss: 1278.163 -> 1278.203\n",
      "\n",
      "-----------------------\n",
      "Start label classifier training\n",
      "Epoch 1 LC iter 1 LC_acc 1.000 loss: inf -> 96.123\n",
      "\n",
      "Epoch 1 LC iter 2 LC_acc 1.000 loss: 96.123 -> 29.383\n",
      "\n",
      "Epoch 1 LC iter 3 LC_acc 1.000 loss: 29.383 -> 14.954\n",
      "\n",
      "Epoch 1 LC iter 4 LC_acc 1.000 loss: 14.954 -> 10.479\n",
      "\n",
      "Epoch 1 LC iter 5 LC_acc 1.000 loss: 10.479 -> 7.720\n",
      "\n",
      "Epoch 1 LC iter 6 LC_acc 1.000 loss: 7.720 -> 6.307\n",
      "\n",
      "Epoch 1 LC iter 7 LC_acc 1.000 loss: 6.307 -> 4.408\n",
      "\n",
      "Epoch 1 LC iter 8 LC_acc 1.000 loss: 4.408 -> 3.198\n",
      "\n",
      "Epoch 1 LC iter 9 LC_acc 1.000 loss: 3.198 -> 2.224\n",
      "\n",
      "Epoch 1 LC iter 10 LC_acc 1.000 loss: 2.224 -> 1.677\n",
      "\n",
      "-----------------------\n",
      "Start feature extractor training\n",
      "Epoch 1 FE iter 1 LC_acc 1.000 DC_acc 0.438 loss: inf -> 2038.187\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.480889\n",
      "\n",
      "Epoch 1 FE iter 2 LC_acc 1.000 DC_acc 0.438 loss: 2038.187 -> 47.950\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.497556\n",
      "\n",
      "Epoch 1 FE iter 3 LC_acc 1.000 DC_acc 0.438 loss: 47.950 -> 30.129\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.499667\n",
      "\n",
      "Epoch 1 FE iter 4 LC_acc 1.000 DC_acc 0.438 loss: 30.129 -> 21.125\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.494778\n",
      "\n",
      "Epoch 1 FE iter 5 LC_acc 1.000 DC_acc 0.438 loss: 21.125 -> 18.614\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.494333\n",
      "\n",
      "Epoch 1 FE iter 6 LC_acc 1.000 DC_acc 0.438 loss: 18.614 -> 16.926\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.501444\n",
      "\n",
      "Epoch 1 FE iter 7 LC_acc 1.000 DC_acc 0.438 loss: 16.926 -> 13.867\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.497556\n",
      "\n",
      "Epoch 1 FE iter 8 LC_acc 1.000 DC_acc 0.438 loss: 13.867 -> 31.172\n",
      "\n",
      "-----------------------\n",
      "Start domain classifier training\n",
      "Epoch 2 DC iter 1 acc 0.438 loss: inf -> 3543.035\n",
      "\n",
      "Epoch 2 DC iter 2 acc 0.438 loss: 3543.035 -> 1275.084\n",
      "\n",
      "Epoch 2 DC iter 3 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 4 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 5 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 6 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 7 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 8 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 9 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 10 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 11 acc 0.438 loss: 1275.084 -> 1278.175\n",
      "\n",
      "Epoch 2 DC iter 12 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 13 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 14 acc 0.438 loss: 1275.084 -> 1278.180\n",
      "\n",
      "Epoch 2 DC iter 15 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 16 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 17 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 18 acc 0.438 loss: 1275.084 -> 1278.202\n",
      "\n",
      "Epoch 2 DC iter 19 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "Epoch 2 DC iter 20 acc 0.438 loss: 1275.084 -> 1278.163\n",
      "\n",
      "-----------------------\n",
      "Start label classifier training\n",
      "Epoch 2 LC iter 1 LC_acc 1.000 loss: inf -> 38.656\n",
      "\n",
      "Epoch 2 LC iter 2 LC_acc 1.000 loss: 38.656 -> 11.346\n",
      "\n",
      "Epoch 2 LC iter 3 LC_acc 1.000 loss: 11.346 -> 4.437\n",
      "\n",
      "Epoch 2 LC iter 4 LC_acc 1.000 loss: 4.437 -> 2.073\n",
      "\n",
      "Epoch 2 LC iter 5 LC_acc 1.000 loss: 2.073 -> 1.129\n",
      "\n",
      "Epoch 2 LC iter 6 LC_acc 1.000 loss: 1.129 -> 0.692\n",
      "\n",
      "Epoch 2 LC iter 7 LC_acc 1.000 loss: 0.692 -> 0.457\n",
      "\n",
      "Epoch 2 LC iter 8 LC_acc 1.000 loss: 0.457 -> 0.315\n",
      "\n",
      "Epoch 2 LC iter 9 LC_acc 1.000 loss: 0.315 -> 0.226\n",
      "\n",
      "-----------------------\n",
      "Start feature extractor training\n",
      "Epoch 2 FE iter 1 LC_acc 1.000 DC_acc 0.438 loss: inf -> 12807.371\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.507556\n",
      "\n",
      "Epoch 2 FE iter 2 LC_acc 1.000 DC_acc 0.438 loss: 12807.371 -> 12790.383\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.511444\n",
      "\n",
      "Epoch 2 FE iter 3 LC_acc 1.000 DC_acc 0.438 loss: 12790.383 -> 12785.314\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.510222\n",
      "\n",
      "Epoch 2 FE iter 4 LC_acc 1.000 DC_acc 0.438 loss: 12785.314 -> 12785.108\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.507444\n",
      "\n",
      "Epoch 2 FE iter 5 LC_acc 1.000 DC_acc 0.438 loss: 12785.108 -> 12784.561\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.520333\n",
      "\n",
      "Epoch 2 FE iter 6 LC_acc 1.000 DC_acc 0.438 loss: 12784.561 -> 12782.074\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.518111\n",
      "\n",
      "Epoch 2 FE iter 7 LC_acc 1.000 DC_acc 0.438 loss: 12782.074 -> 12782.705\n",
      "\n",
      "-----------------------\n",
      "Start domain classifier training\n",
      "Epoch 3 DC iter 1 acc 0.438 loss: inf -> 1278.163\n",
      "\n",
      "Epoch 3 DC iter 2 acc 0.438 loss: 1278.163 -> 1278.264\n",
      "\n",
      "Epoch 3 DC iter 3 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 3 DC iter 4 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 3 DC iter 5 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 3 DC iter 6 acc 0.438 loss: 1278.163 -> 1278.165\n",
      "\n",
      "Epoch 3 DC iter 7 acc 0.438 loss: 1278.163 -> 1278.166\n",
      "\n",
      "Epoch 3 DC iter 8 acc 0.438 loss: 1278.163 -> 1278.165\n",
      "\n",
      "Epoch 3 DC iter 9 acc 0.438 loss: 1278.163 -> 1278.166\n",
      "\n",
      "Epoch 3 DC iter 10 acc 0.438 loss: 1278.163 -> 1278.165\n",
      "\n",
      "Epoch 3 DC iter 11 acc 0.438 loss: 1278.163 -> 1278.166\n",
      "\n",
      "Epoch 3 DC iter 12 acc 0.438 loss: 1278.163 -> 1278.165\n",
      "\n",
      "Epoch 3 DC iter 13 acc 0.438 loss: 1278.163 -> 1278.165\n",
      "\n",
      "Epoch 3 DC iter 14 acc 0.438 loss: 1278.163 -> 1278.173\n",
      "\n",
      "Epoch 3 DC iter 15 acc 0.438 loss: 1278.163 -> 1278.163\n",
      "\n",
      "Epoch 3 DC iter 16 acc 0.438 loss: 1278.163 -> 1278.167\n",
      "\n",
      "Epoch 3 DC iter 17 acc 0.438 loss: 1278.163 -> 1278.166\n",
      "\n",
      "Epoch 3 DC iter 18 acc 0.438 loss: 1278.163 -> 1278.166\n",
      "\n",
      "Epoch 3 DC iter 19 acc 0.438 loss: 1278.163 -> 1278.165\n",
      "\n",
      "Epoch 3 DC iter 20 acc 0.438 loss: 1278.163 -> 1278.166\n",
      "\n",
      "-----------------------\n",
      "Start label classifier training\n",
      "Epoch 3 LC iter 1 LC_acc 1.000 loss: inf -> 27.074\n",
      "\n",
      "Epoch 3 LC iter 2 LC_acc 1.000 loss: 27.074 -> 5.455\n",
      "\n",
      "Epoch 3 LC iter 3 LC_acc 1.000 loss: 5.455 -> 1.282\n",
      "\n",
      "Epoch 3 LC iter 4 LC_acc 1.000 loss: 1.282 -> 0.364\n",
      "\n",
      "Epoch 3 LC iter 5 LC_acc 1.000 loss: 0.364 -> 0.136\n",
      "\n",
      "Epoch 3 LC iter 6 LC_acc 1.000 loss: 0.136 -> 0.080\n",
      "\n",
      "-----------------------\n",
      "Start feature extractor training\n",
      "Epoch 3 FE iter 1 LC_acc 1.000 DC_acc 0.438 loss: inf -> 12847.050\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.517444\n",
      "\n",
      "Epoch 3 FE iter 2 LC_acc 1.000 DC_acc 0.438 loss: 12847.050 -> 12830.796\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.514222\n",
      "\n",
      "Epoch 3 FE iter 3 LC_acc 1.000 DC_acc 0.438 loss: 12830.796 -> 12828.867\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label classifier accuracy on MNIST-M test set (DA): 0.510889\n",
      "\n",
      "Epoch 3 FE iter 4 LC_acc 1.000 DC_acc 0.438 loss: 12828.867 -> 12827.762\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.511778\n",
      "\n",
      "Epoch 3 FE iter 5 LC_acc 1.000 DC_acc 0.438 loss: 12827.762 -> 12827.405\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.512778\n",
      "\n",
      "Epoch 3 FE iter 6 LC_acc 1.000 DC_acc 0.438 loss: 12827.405 -> 12827.350\n",
      "\n",
      "-----------------------\n",
      "Start domain classifier training\n",
      "Epoch 4 DC iter 1 acc 0.438 loss: inf -> 1278.175\n",
      "\n",
      "Epoch 4 DC iter 2 acc 0.438 loss: 1278.175 -> 1278.163\n",
      "\n",
      "Epoch 4 DC iter 3 acc 0.438 loss: 1278.175 -> 1278.163\n",
      "\n",
      "Epoch 4 DC iter 4 acc 0.438 loss: 1278.175 -> 1278.184\n",
      "\n",
      "Epoch 4 DC iter 5 acc 0.438 loss: 1278.175 -> 1278.163\n",
      "\n",
      "Epoch 4 DC iter 6 acc 0.438 loss: 1278.175 -> 1278.163\n",
      "\n",
      "Epoch 4 DC iter 7 acc 0.438 loss: 1278.175 -> 1278.173\n",
      "\n",
      "Epoch 4 DC iter 8 acc 0.438 loss: 1278.175 -> 1278.163\n",
      "\n",
      "Epoch 4 DC iter 9 acc 0.438 loss: 1278.175 -> 1278.167\n",
      "\n",
      "Epoch 4 DC iter 10 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 4 DC iter 11 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 4 DC iter 12 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 4 DC iter 13 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 4 DC iter 14 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 4 DC iter 15 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 4 DC iter 16 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 4 DC iter 17 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 4 DC iter 18 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 4 DC iter 19 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 4 DC iter 20 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "-----------------------\n",
      "Start label classifier training\n",
      "Epoch 4 LC iter 1 LC_acc 1.000 loss: inf -> 27.329\n",
      "\n",
      "Epoch 4 LC iter 2 LC_acc 1.000 loss: 27.329 -> 6.418\n",
      "\n",
      "Epoch 4 LC iter 3 LC_acc 1.000 loss: 6.418 -> 1.026\n",
      "\n",
      "Epoch 4 LC iter 4 LC_acc 1.000 loss: 1.026 -> 0.317\n",
      "\n",
      "Epoch 4 LC iter 5 LC_acc 1.000 loss: 0.317 -> 0.113\n",
      "\n",
      "Epoch 4 LC iter 6 LC_acc 1.000 loss: 0.113 -> 0.039\n",
      "\n",
      "-----------------------\n",
      "Start feature extractor training\n",
      "Epoch 4 FE iter 1 LC_acc 1.000 DC_acc 0.438 loss: inf -> 12804.283\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.528111\n",
      "\n",
      "Epoch 4 FE iter 2 LC_acc 1.000 DC_acc 0.438 loss: 12804.283 -> 12789.818\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.535333\n",
      "\n",
      "Epoch 4 FE iter 3 LC_acc 1.000 DC_acc 0.438 loss: 12789.818 -> 12788.143\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.529444\n",
      "\n",
      "Epoch 4 FE iter 4 LC_acc 1.000 DC_acc 0.438 loss: 12788.143 -> 12787.609\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.525444\n",
      "\n",
      "Epoch 4 FE iter 5 LC_acc 1.000 DC_acc 0.438 loss: 12787.609 -> 12786.688\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.531889\n",
      "\n",
      "Epoch 4 FE iter 6 LC_acc 1.000 DC_acc 0.438 loss: 12786.688 -> 12785.948\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.526556\n",
      "\n",
      "Epoch 4 FE iter 7 LC_acc 1.000 DC_acc 0.438 loss: 12785.948 -> 12785.844\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.529667\n",
      "\n",
      "Epoch 4 FE iter 8 LC_acc 1.000 DC_acc 0.438 loss: 12785.844 -> 12785.832\n",
      "\n",
      "-----------------------\n",
      "Start domain classifier training\n",
      "Epoch 5 DC iter 1 acc 0.438 loss: inf -> 1278.176\n",
      "\n",
      "Epoch 5 DC iter 2 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 3 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 4 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 5 acc 0.438 loss: 1278.176 -> 1278.214\n",
      "\n",
      "Epoch 5 DC iter 6 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 7 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 8 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 9 acc 0.438 loss: 1278.176 -> 1278.174\n",
      "\n",
      "Epoch 5 DC iter 10 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 11 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 12 acc 0.438 loss: 1278.176 -> 1278.185\n",
      "\n",
      "Epoch 5 DC iter 13 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 14 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 15 acc 0.438 loss: 1278.176 -> 1278.172\n",
      "\n",
      "Epoch 5 DC iter 16 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 17 acc 0.438 loss: 1278.176 -> 1278.171\n",
      "\n",
      "Epoch 5 DC iter 18 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 5 DC iter 19 acc 0.438 loss: 1278.176 -> 1278.170\n",
      "\n",
      "Epoch 5 DC iter 20 acc 0.438 loss: 1278.176 -> 1278.164\n",
      "\n",
      "-----------------------\n",
      "Start label classifier training\n",
      "Epoch 5 LC iter 1 LC_acc 1.000 loss: inf -> 28.260\n",
      "\n",
      "Epoch 5 LC iter 2 LC_acc 1.000 loss: 28.260 -> 5.573\n",
      "\n",
      "Epoch 5 LC iter 3 LC_acc 1.000 loss: 5.573 -> 1.296\n",
      "\n",
      "Epoch 5 LC iter 4 LC_acc 1.000 loss: 1.296 -> 0.380\n",
      "\n",
      "Epoch 5 LC iter 5 LC_acc 1.000 loss: 0.380 -> 0.171\n",
      "\n",
      "Epoch 5 LC iter 6 LC_acc 1.000 loss: 0.171 -> 0.036\n",
      "\n",
      "Epoch 5 LC iter 7 LC_acc 1.000 loss: 0.036 -> 0.019\n",
      "\n",
      "-----------------------\n",
      "Start feature extractor training\n",
      "Epoch 5 FE iter 1 LC_acc 1.000 DC_acc 0.438 loss: inf -> 12801.886\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.532000\n",
      "\n",
      "Epoch 5 FE iter 2 LC_acc 1.000 DC_acc 0.438 loss: 12801.886 -> 12791.827\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.527111\n",
      "\n",
      "Epoch 5 FE iter 3 LC_acc 1.000 DC_acc 0.438 loss: 12791.827 -> 12785.184\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.541778\n",
      "\n",
      "Epoch 5 FE iter 4 LC_acc 1.000 DC_acc 0.438 loss: 12785.184 -> 12784.824\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.538111\n",
      "\n",
      "Epoch 5 FE iter 5 LC_acc 1.000 DC_acc 0.438 loss: 12784.824 -> 12784.305\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.533222\n",
      "\n",
      "Epoch 5 FE iter 6 LC_acc 1.000 DC_acc 0.438 loss: 12784.305 -> 12784.268\n",
      "\n",
      "-----------------------\n",
      "Start domain classifier training\n",
      "Epoch 6 DC iter 1 acc 0.438 loss: inf -> 1278.176\n",
      "\n",
      "Epoch 6 DC iter 2 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 3 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 4 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 5 acc 0.438 loss: 1278.176 -> 1278.211\n",
      "\n",
      "Epoch 6 DC iter 6 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 7 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 8 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 9 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 10 acc 0.438 loss: 1278.176 -> 1278.203\n",
      "\n",
      "Epoch 6 DC iter 11 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 12 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 13 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 14 acc 0.438 loss: 1278.176 -> 1278.182\n",
      "\n",
      "Epoch 6 DC iter 15 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 16 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 17 acc 0.438 loss: 1278.176 -> 1278.176\n",
      "\n",
      "Epoch 6 DC iter 18 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 6 DC iter 19 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 6 DC iter 20 acc 0.438 loss: 1278.176 -> 1278.166\n",
      "\n",
      "-----------------------\n",
      "Start label classifier training\n",
      "Epoch 6 LC iter 1 LC_acc 1.000 loss: inf -> 29.002\n",
      "\n",
      "Epoch 6 LC iter 2 LC_acc 1.000 loss: 29.002 -> 5.987\n",
      "\n",
      "Epoch 6 LC iter 3 LC_acc 1.000 loss: 5.987 -> 1.428\n",
      "\n",
      "Epoch 6 LC iter 4 LC_acc 1.000 loss: 1.428 -> 0.272\n",
      "\n",
      "Epoch 6 LC iter 5 LC_acc 1.000 loss: 0.272 -> 0.061\n",
      "\n",
      "Epoch 6 LC iter 6 LC_acc 1.000 loss: 0.061 -> 0.027\n",
      "\n",
      "-----------------------\n",
      "Start feature extractor training\n",
      "Epoch 6 FE iter 1 LC_acc 1.000 DC_acc 0.438 loss: inf -> 12795.936\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.531222\n",
      "\n",
      "Epoch 6 FE iter 2 LC_acc 1.000 DC_acc 0.438 loss: 12795.936 -> 12783.596\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.538222\n",
      "\n",
      "Epoch 6 FE iter 3 LC_acc 1.000 DC_acc 0.438 loss: 12783.596 -> 12781.589\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.541778\n",
      "\n",
      "Epoch 6 FE iter 4 LC_acc 1.000 DC_acc 0.438 loss: 12781.589 -> 12781.206\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.545889\n",
      "\n",
      "Epoch 6 FE iter 5 LC_acc 1.000 DC_acc 0.438 loss: 12781.206 -> 12780.341\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.546444\n",
      "\n",
      "Epoch 6 FE iter 6 LC_acc 1.000 DC_acc 0.438 loss: 12780.341 -> 12780.290\n",
      "\n",
      "-----------------------\n",
      "Start domain classifier training\n",
      "Epoch 7 DC iter 1 acc 0.438 loss: inf -> 1278.176\n",
      "\n",
      "Epoch 7 DC iter 2 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 7 DC iter 3 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 7 DC iter 4 acc 0.438 loss: 1278.176 -> 1278.186\n",
      "\n",
      "Epoch 7 DC iter 5 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 7 DC iter 6 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7 DC iter 7 acc 0.438 loss: 1278.176 -> 1278.171\n",
      "\n",
      "Epoch 7 DC iter 8 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 7 DC iter 9 acc 0.438 loss: 1278.176 -> 1278.172\n",
      "\n",
      "Epoch 7 DC iter 10 acc 0.438 loss: 1278.176 -> 1278.164\n",
      "\n",
      "Epoch 7 DC iter 11 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 7 DC iter 12 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 7 DC iter 13 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 7 DC iter 14 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 7 DC iter 15 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 7 DC iter 16 acc 0.438 loss: 1278.176 -> 1278.166\n",
      "\n",
      "Epoch 7 DC iter 17 acc 0.438 loss: 1278.176 -> 1278.166\n",
      "\n",
      "Epoch 7 DC iter 18 acc 0.438 loss: 1278.176 -> 1278.166\n",
      "\n",
      "Epoch 7 DC iter 19 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 7 DC iter 20 acc 0.438 loss: 1278.176 -> 1278.166\n",
      "\n",
      "-----------------------\n",
      "Start label classifier training\n",
      "Epoch 7 LC iter 1 LC_acc 1.000 loss: inf -> 24.658\n",
      "\n",
      "Epoch 7 LC iter 2 LC_acc 1.000 loss: 24.658 -> 6.388\n",
      "\n",
      "Epoch 7 LC iter 3 LC_acc 1.000 loss: 6.388 -> 1.626\n",
      "\n",
      "Epoch 7 LC iter 4 LC_acc 1.000 loss: 1.626 -> 0.195\n",
      "\n",
      "Epoch 7 LC iter 5 LC_acc 1.000 loss: 0.195 -> 0.028\n",
      "\n",
      "Epoch 7 LC iter 6 LC_acc 1.000 loss: 0.028 -> 0.021\n",
      "\n",
      "-----------------------\n",
      "Start feature extractor training\n",
      "Epoch 7 FE iter 1 LC_acc 1.000 DC_acc 0.438 loss: inf -> 12794.494\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.527778\n",
      "\n",
      "Epoch 7 FE iter 2 LC_acc 1.000 DC_acc 0.438 loss: 12794.494 -> 12783.644\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.540333\n",
      "\n",
      "Epoch 7 FE iter 3 LC_acc 1.000 DC_acc 0.438 loss: 12783.644 -> 12784.009\n",
      "\n",
      "-----------------------\n",
      "Start domain classifier training\n",
      "Epoch 8 DC iter 1 acc 0.438 loss: inf -> 1278.175\n",
      "\n",
      "Epoch 8 DC iter 2 acc 0.438 loss: 1278.175 -> 1278.163\n",
      "\n",
      "Epoch 8 DC iter 3 acc 0.438 loss: 1278.175 -> 1278.163\n",
      "\n",
      "Epoch 8 DC iter 4 acc 0.438 loss: 1278.175 -> 1278.163\n",
      "\n",
      "Epoch 8 DC iter 5 acc 0.438 loss: 1278.175 -> 1278.208\n",
      "\n",
      "Epoch 8 DC iter 6 acc 0.438 loss: 1278.175 -> 1278.163\n",
      "\n",
      "Epoch 8 DC iter 7 acc 0.438 loss: 1278.175 -> 1278.163\n",
      "\n",
      "Epoch 8 DC iter 8 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 8 DC iter 9 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 8 DC iter 10 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 8 DC iter 11 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 8 DC iter 12 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 8 DC iter 13 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 8 DC iter 14 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 8 DC iter 15 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 8 DC iter 16 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 8 DC iter 17 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "Epoch 8 DC iter 18 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 8 DC iter 19 acc 0.438 loss: 1278.175 -> 1278.166\n",
      "\n",
      "Epoch 8 DC iter 20 acc 0.438 loss: 1278.175 -> 1278.165\n",
      "\n",
      "-----------------------\n",
      "Start label classifier training\n",
      "Epoch 8 LC iter 1 LC_acc 1.000 loss: inf -> 22.787\n",
      "\n",
      "Epoch 8 LC iter 2 LC_acc 1.000 loss: 22.787 -> 7.234\n",
      "\n",
      "Epoch 8 LC iter 3 LC_acc 1.000 loss: 7.234 -> 0.788\n",
      "\n",
      "Epoch 8 LC iter 4 LC_acc 1.000 loss: 0.788 -> 0.092\n",
      "\n",
      "Epoch 8 LC iter 5 LC_acc 1.000 loss: 0.092 -> 0.029\n",
      "\n",
      "-----------------------\n",
      "Start feature extractor training\n",
      "Epoch 8 FE iter 1 LC_acc 1.000 DC_acc 0.438 loss: inf -> 12804.219\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.541444\n",
      "\n",
      "Epoch 8 FE iter 2 LC_acc 1.000 DC_acc 0.438 loss: 12804.219 -> 12796.677\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.538667\n",
      "\n",
      "Epoch 8 FE iter 3 LC_acc 1.000 DC_acc 0.438 loss: 12796.677 -> 12793.530\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.539333\n",
      "\n",
      "Epoch 8 FE iter 4 LC_acc 1.000 DC_acc 0.438 loss: 12793.530 -> 12792.727\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.535444\n",
      "\n",
      "Epoch 8 FE iter 5 LC_acc 1.000 DC_acc 0.438 loss: 12792.727 -> 12792.609\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.535556\n",
      "\n",
      "Epoch 8 FE iter 6 LC_acc 1.000 DC_acc 0.438 loss: 12792.609 -> 12792.601\n",
      "\n",
      "-----------------------\n",
      "Start domain classifier training\n",
      "Epoch 9 DC iter 1 acc 0.438 loss: inf -> 1278.176\n",
      "\n",
      "Epoch 9 DC iter 2 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 9 DC iter 3 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 9 DC iter 4 acc 0.438 loss: 1278.176 -> 1278.186\n",
      "\n",
      "Epoch 9 DC iter 5 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 9 DC iter 6 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 9 DC iter 7 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 9 DC iter 8 acc 0.438 loss: 1278.176 -> 1278.190\n",
      "\n",
      "Epoch 9 DC iter 9 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 9 DC iter 10 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 9 DC iter 11 acc 0.438 loss: 1278.176 -> 1278.168\n",
      "\n",
      "Epoch 9 DC iter 12 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 9 DC iter 13 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 9 DC iter 14 acc 0.438 loss: 1278.176 -> 1278.166\n",
      "\n",
      "Epoch 9 DC iter 15 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 9 DC iter 16 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 9 DC iter 17 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 9 DC iter 18 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 9 DC iter 19 acc 0.438 loss: 1278.176 -> 1278.165\n",
      "\n",
      "Epoch 9 DC iter 20 acc 0.438 loss: 1278.176 -> 1278.166\n",
      "\n",
      "-----------------------\n",
      "Start label classifier training\n",
      "Epoch 9 LC iter 1 LC_acc 1.000 loss: inf -> 26.329\n",
      "\n",
      "Epoch 9 LC iter 2 LC_acc 1.000 loss: 26.329 -> 4.069\n",
      "\n",
      "Epoch 9 LC iter 3 LC_acc 1.000 loss: 4.069 -> 1.053\n",
      "\n",
      "Epoch 9 LC iter 4 LC_acc 1.000 loss: 1.053 -> 0.157\n",
      "\n",
      "Epoch 9 LC iter 5 LC_acc 1.000 loss: 0.157 -> 0.015\n",
      "\n",
      "Epoch 9 LC iter 6 LC_acc 1.000 loss: 0.015 -> 0.010\n",
      "\n",
      "-----------------------\n",
      "Start feature extractor training\n",
      "Epoch 9 FE iter 1 LC_acc 1.000 DC_acc 0.438 loss: inf -> 12772.316\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.503778\n",
      "\n",
      "Epoch 9 FE iter 2 LC_acc 1.000 DC_acc 0.438 loss: 12772.316 -> 12764.472\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.529111\n",
      "\n",
      "Epoch 9 FE iter 3 LC_acc 1.000 DC_acc 0.438 loss: 12764.472 -> 12761.907\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.532556\n",
      "\n",
      "Epoch 9 FE iter 4 LC_acc 1.000 DC_acc 0.438 loss: 12761.907 -> 12761.757\n",
      "\n",
      "Label classifier accuracy on MNIST-M test set (DA): 0.534111\n",
      "\n",
      "Epoch 9 FE iter 5 LC_acc 1.000 DC_acc 0.438 loss: 12761.757 -> 12761.741\n",
      "\n",
      "-----------------------\n",
      "Start domain classifier training\n",
      "Epoch 10 DC iter 1 acc 0.438 loss: inf -> 1278.176\n",
      "\n",
      "Epoch 10 DC iter 2 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 10 DC iter 3 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 10 DC iter 4 acc 0.438 loss: 1278.176 -> 1278.186\n",
      "\n",
      "Epoch 10 DC iter 5 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 10 DC iter 6 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n",
      "Epoch 10 DC iter 7 acc 0.438 loss: 1278.176 -> 1278.163\n",
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-65-73d9ae7a0dfe>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m             \u001b[0mloss_DC\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m             \u001b[0moptimizer_DC\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     98\u001b[0m                     \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_exp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m                     \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m                 \u001b[0mbias_correction1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m \u001b[0;34m**\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'step'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#%%skip $load_model_da_dc\n",
    "\n",
    "\n",
    "total_epoch = 25\n",
    "lamda = 10\n",
    "\n",
    "\n",
    "prev_loss_DC = np.float(\"inf\")\n",
    "prev_loss_LC = np.float(\"inf\")\n",
    "prev_loss_FE = np.float(\"inf\")\n",
    "\n",
    "for epoch in range(total_epoch):\n",
    "    net.train()\n",
    "    p = epoch * 1.0 / total_epoch\n",
    "\n",
    "    \n",
    "    print('-----------------------')\n",
    "    print('Start domain classifier training') \n",
    "    lr_DC = 0.01\n",
    "    optimizer_DC = optim.Adam(params_DC, lr=0.01)\n",
    "    total_DC_iter = 20\n",
    "    # domain classifier \n",
    "    prev_loss_DC = np.float(\"inf\")\n",
    "    for DC_iter in range(total_DC_iter):\n",
    "        DC_p = DC_iter * 1.0 / total_DC_iter\n",
    "        epoch_loss_DC = 0.0\n",
    "        running_loss_DC = 0.0\n",
    "        #adjust_lr(optimizer_DC, DC_p, lr_DC)\n",
    "        for batch_idx, data in enumerate(trainloader_da):\n",
    "        \n",
    "            # get inputs\n",
    "            source_size = data[0].size()[0] // 2\n",
    "            inputs, labels, domains = data\n",
    "\n",
    "            domains = domains.to(torch.float32)\n",
    "            if (use_gpu):\n",
    "                inputs, domains = inputs.cuda(), domains.cuda()\n",
    "            inputs, domains = Variable(inputs), Variable(domains)\n",
    "\n",
    "            # forward\n",
    "            source_domain = domains[:source_size]\n",
    "            target_domain = domains[-source_size:]\n",
    "            \n",
    "            optimizer_DC.zero_grad()\n",
    "            _, pred_domains = net(inputs)\n",
    "            pred_source_domain = pred_domains[:source_size]\n",
    "            pred_target_domain = pred_domains[-source_size:]            \n",
    "            \n",
    "            # pred_source_domains should be 0 \n",
    "            loss_DC_source = criterion_DC(pred_source_domain , source_domain)\n",
    "            # pred_source_domains should be 1\n",
    "            loss_DC_target = criterion_DC(pred_target_domain , target_domain)\n",
    "            \n",
    "            \n",
    "            loss_DC = loss_DC_source + loss_DC_target\n",
    "\n",
    "            loss_DC.backward()\n",
    "            optimizer_DC.step()\n",
    "            \n",
    "            \n",
    "            pred_source_domain_binary = (pred_source_domain > 0.5).float() * 1\n",
    "            pred_target_domain_binary = (pred_target_domain > 0.5).float() * 1 \n",
    "            \n",
    "            DC_acc = (source_domain == pred_source_domain_binary[:,0]).sum() + \\\n",
    "                     (target_domain == pred_target_domain_binary[:,0]).sum()\n",
    "            DC_acc = DC_acc.cpu().data.numpy()/batch_size\n",
    "\n",
    "            # stat\n",
    "            epoch_loss_DC += loss_DC.item()\n",
    "            running_loss_DC += loss_DC.item()\n",
    "            #print(loss_DC.cpu().data.numpy())\n",
    "\n",
    "        print(\"Epoch %d DC iter %d acc %.3f loss: %.3f -> %.3f\\n\" % (epoch , DC_iter + 1, DC_acc, prev_loss_DC, epoch_loss_DC))\n",
    "        if prev_loss_DC - epoch_loss_DC < 0.1 : #or prev_loss_DC < epoch_loss_DC:\n",
    "            prev_loss_DC = epoch_loss_DC   \n",
    "        else:\n",
    "            prev_loss_DC = epoch_loss_DC                \n",
    "\n",
    "    \n",
    " \n",
    "\n",
    "    print('-----------------------')\n",
    "    print('Start label classifier training')\n",
    "    prev_loss_LC = np.float(\"inf\")\n",
    "    # Label classifier\n",
    "    lr_LC = 0.001\n",
    "    optimizer_LC = optim.Adam(params_LC, lr=lr_LC)\n",
    "    total_LC_iter = 5\n",
    "    for LC_iter in range(10):\n",
    "        LC_p = LC_iter * 1.0 / total_LC_iter\n",
    "        adjust_lr(optimizer_LC, LC_p, lr_LC)\n",
    "        epoch_loss_LC = 0.0\n",
    "        running_loss_LC = 0.0\n",
    "        for batch_idx, data in enumerate(trainloader_da):\n",
    "\n",
    "            # get inputs\n",
    "            source_size = data[0].size()[0] // 2\n",
    "\n",
    "            inputs, labels, domains = data\n",
    "\n",
    "            domains = domains.to(torch.float32)\n",
    "            if (use_gpu):\n",
    "                inputs, labels, domains = inputs.cuda(), labels.cuda(), domains.cuda()\n",
    "            inputs, labels, domains = Variable(inputs), Variable(labels), Variable(domains)\n",
    "\n",
    "            optimizer_LC.zero_grad()\n",
    "            # forward\n",
    "            pred_labels, _ = net(inputs)\n",
    "            src_label_logit = pred_labels[:source_size]\n",
    "            loss_LC = criterion_LC(src_label_logit, labels[:source_size])\n",
    "            loss_LC.backward()\n",
    "            optimizer_LC.step()\n",
    "\n",
    "            _, src_label = torch.max(src_label_logit, dim = 1)\n",
    "\n",
    "            LC_acc = (src_label == labels[:source_size]).sum()            \n",
    "            LC_acc = LC_acc.cpu().data.numpy()/source_size\n",
    "            \n",
    "            # stat\n",
    "            epoch_loss_LC += loss_LC.item()\n",
    "            running_loss_LC += loss_LC.item()\n",
    "\n",
    "        print(\"Epoch %d LC iter %d LC_acc %.3f loss: %.3f -> %.3f\\n\" % (epoch, LC_iter + 1, LC_acc , prev_loss_LC, epoch_loss_LC))\n",
    "        if prev_loss_LC - epoch_loss_LC < 0.1 : #or prev_loss_LC < epoch_loss_LC:\n",
    "            #prev_loss_LC = epoch_loss_LC  \n",
    "            break\n",
    "        else:\n",
    "            prev_loss_LC = epoch_loss_LC            \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    print('-----------------------')\n",
    "    print('Start feature extractor training')\n",
    "    # feature extractor\n",
    "    prev_loss_FE = np.float(\"inf\")\n",
    "    lr_FE = 0.001\n",
    "    optimizer_FE = optim.Adam(params_FE, lr=lr_FE)\n",
    "    total_FE_iter = 10\n",
    "    for FE_iter in range(total_FE_iter):\n",
    "        FE_p = FE_iter * 1.0 / total_FE_iter\n",
    "        adjust_lr(optimizer_FE, FE_p, lr_FE)\n",
    "        epoch_loss_FE = 0.0\n",
    "        running_loss_FE = 0.0\n",
    "        for batch_idx, data in enumerate(trainloader_da):\n",
    "\n",
    "            # get inputs\n",
    "            source_size = data[0].size()[0] // 2\n",
    "\n",
    "            inputs, labels, domains = data\n",
    "\n",
    "            domains = domains.to(torch.float32)\n",
    "            if (use_gpu):\n",
    "                inputs, labels, domains = inputs.cuda(), labels.cuda(), domains.cuda()\n",
    "            inputs, labels, domains = Variable(inputs), Variable(labels), Variable(domains)\n",
    "\n",
    "            optimizer_FE.zero_grad()\n",
    "            # forward\n",
    "            source_domain = domains[:source_size]\n",
    "            target_domain = domains[-source_size:]\n",
    "            \n",
    "            pred_labels, pred_domains = net(inputs)\n",
    "            pred_source_domain = pred_domains[:source_size]\n",
    "            pred_target_domain = pred_domains[-source_size:]\n",
    "            src_label_logit = pred_labels[:source_size]\n",
    "            loss_LC = criterion_LC(src_label_logit, labels[:source_size])\n",
    "            # pred_source_domains should be 0          \n",
    "            loss_DC_source = criterion_DC(pred_source_domain , source_domain)\n",
    "            # pred_target_domains should be 1\n",
    "            loss_DC_target = criterion_DC(pred_target_domain , source_domain)\n",
    "            \n",
    "            loss_FE = loss_LC + lamda*(loss_DC_source + loss_DC_target)\n",
    "            loss_FE.backward()\n",
    "            optimizer_FE.step()\n",
    "\n",
    "\n",
    "            pred_source_domain_binary = (pred_source_domain > 0.5).float() * 1\n",
    "            pred_target_domain_binary = (pred_target_domain > 0.5).float() * 1 \n",
    "            \n",
    "            \n",
    "            _, src_label = torch.max(src_label_logit, dim = 1)\n",
    "\n",
    "            LC_acc = (src_label == labels[:source_size]).sum()            \n",
    "            LC_acc = LC_acc.cpu().data.numpy()/source_size\n",
    "            \n",
    "            \n",
    "            DC_acc = (source_domain == pred_source_domain_binary[:,0]).sum() + \\\n",
    "                     (target_domain == pred_target_domain_binary[:,0]).sum()\n",
    "            DC_acc = DC_acc.cpu().data.numpy()/batch_size            \n",
    "            \n",
    "            # stat\n",
    "            epoch_loss_FE += loss_FE.item()\n",
    "            running_loss_FE += loss_FE.item()\n",
    "\n",
    "        print(\"Epoch %d FE iter %d LC_acc %.3f DC_acc %.3f loss: %.3f -> %.3f\\n\" % (epoch , FE_iter + 1, LC_acc, \\\n",
    "                                                                                    DC_acc, prev_loss_FE, epoch_loss_FE))\n",
    "        if prev_loss_FE - epoch_loss_FE < 0.1 :#or prev_loss_FE < epoch_loss_FE:\n",
    "            break\n",
    "        else:\n",
    "            prev_loss_FE = epoch_loss_FE            \n",
    "        print(\"Label classifier accuracy on MNIST-M test set (DA): %f\\n\"\n",
    "          %evaluate_da_accuracy(net, testloader_m, source=True))\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(net.state_dict(), para_file_da)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test on MNIST and MNIST-M dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:11: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label classifier accuracy on MNIST test set (DA): 0.991300\n"
     ]
    }
   ],
   "source": [
    "print(\"Label classifier accuracy on MNIST test set (DA): %f\"\n",
    "      %evaluate_da_accuracy(net, testloader, source=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:11: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label classifier accuracy on MNIST-M test set (DA): 0.534000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Label classifier accuracy on MNIST-M test set (DA): %f\\n\"\n",
    "      %evaluate_da_accuracy(net, testloader_m, source=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "My_CNN_Tutorial.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}