tasfia_DAMC_TMM2020.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['aeroplane', 'bicycle', 'bus', 'car', 'horse', 'knife', 'motorcycle', 'person', 'plant', 'skateboard', 'train', 'truck']\n",
      "['aeroplane', 'bicycle', 'bus', 'car', 'horse', 'knife', 'motorcycle', 'person', 'plant', 'skateboard', 'train', 'truck']\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "# source and target list creator\n",
    "import os\n",
    "import random\n",
    "p_path = os.path.join('C:/PHD/data/train')\n",
    "dir_list = os.listdir(p_path)\n",
    "print(dir_list)\n",
    "\n",
    "class_list = [\"bicycle\", \"bus\", \"car\", \"motorcycle\",\"train\",\"truck\", \"unk\"]\n",
    "path_source = \"C:/PHD/data/source1_list.txt\"\n",
    "path_target = \"C:/PHD/data/target1_list.txt\"\n",
    "\n",
    "write_source = open(path_source,\"w\")\n",
    "write_target = open(path_target,\"w\")\n",
    "for k, direc in enumerate(dir_list):\n",
    "    if not '.txt' in direc:\n",
    "        files = os.listdir(os.path.join(p_path, direc))\n",
    "        for i, file in enumerate(files):\n",
    "            if direc in class_list:\n",
    "                class_name = direc\n",
    "                file_name = os.path.join(p_path, direc, file)\n",
    "                write_source.write('%s %s\\n' % (file_name, class_list.index(class_name)))\n",
    "            else:\n",
    "                continue\n",
    "p_path = os.path.join('C:/PHD/data/validation')\n",
    "dir_list = os.listdir(p_path)\n",
    "print(dir_list)\n",
    "for k, direc in enumerate(dir_list):\n",
    "    if not '.txt' in direc:\n",
    "        files = os.listdir(os.path.join(p_path, direc))\n",
    "        for i, file in enumerate(files):\n",
    "            if direc in class_list:\n",
    "                class_name = direc\n",
    "            else:\n",
    "                class_name = \"unk\"\n",
    "            file_name = os.path.join(p_path, direc, file)\n",
    "            write_target.write('%s %s\\n' % (file_name, class_list.index(class_name)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\tshermin\\AppData\\Local\\Continuum\\lib\\site-packages\\torchvision\\transforms\\transforms.py:188: UserWarning: The use of the transforms.Scale transform is deprecated, please use transforms.Resize instead.\n",
      "  \"please use transforms.Resize instead.\")\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "from easydict import EasyDict\n",
    "import argparse\n",
    "from utils.utils import *\n",
    "import torch.nn as nn \n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torchvision.transforms as transforms\n",
    "from data_loader.get_loader import get_loader\n",
    "import numpy as np\n",
    "import os\n",
    "from torchvision import models\n",
    "\n",
    "# Training settings\n",
    "parser = argparse.ArgumentParser(description='PyTorch Open-set DAMC')\n",
    "args = parser.parse_known_args()[0]\n",
    "args = EasyDict({\n",
    "    \"batch_size\":16,\n",
    "    \"epochs\": 1000,\n",
    "    \"log_interval\" :100,\n",
    "    \"lr\":0.001,\n",
    "    \"net\":'resnet152',\n",
    "    \"save\": True,\n",
    "    \"save_path\":\"C:/PHD/data/OPDA_BP/\",\n",
    "    \"source_path\":\"C:/PHD/data/source1_list.txt\",\n",
    "    \"target_path\":\"C:/PHD/data/target1_list.txt\",\n",
    "    \"seed\":1,\n",
    "    \"unit_size\":1000,\n",
    "    \"update_lower\": False,\n",
    "    \"no_cuda\":False\n",
    "    \n",
    "})\n",
    "\n",
    "#path_source = \"C:/PHD/data/ETN/data/office/WDsource_list.txt\"\n",
    "#path_target = \"C:/PHD/data/ETN/data/office/WDtarget_list.txt\"\n",
    "#\"source_path\":\"C:/PHD/data/source_list.txt\",\n",
    "#    \"target_path\":\"C:/PHD/data/target_list.txt\",\n",
    "\n",
    "args.cuda =  torch.cuda.is_available()\n",
    "\n",
    "\n",
    "source_data = args.source_path\n",
    "target_data = args.target_path\n",
    "evaluation_data = args.target_path\n",
    "batch_size = args.batch_size\n",
    "\n",
    "data_transforms = {\n",
    "    source_data: transforms.Compose([\n",
    "        transforms.Scale(256),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.RandomCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    target_data: transforms.Compose([\n",
    "        transforms.Scale(256),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.RandomCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    evaluation_data: transforms.Compose([\n",
    "        transforms.Scale(256),\n",
    "        transforms.CenterCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "}\n",
    "use_gpu = torch.cuda.is_available()\n",
    "torch.manual_seed(args.seed)\n",
    "if args.cuda:\n",
    "    torch.cuda.manual_seed(args.seed)\n",
    "train_loader, test_loader = get_loader(source_data, target_data, evaluation_data,\n",
    "                                       data_transforms, batch_size=args.batch_size)\n",
    "dataset_train = train_loader.load_data()\n",
    "dataset_test = test_loader\n",
    "\n",
    "num_class = 7\n",
    "#class_list = [\"back_pack\", \"bike\", \"bike_helmet\", \"bookcase\", \"bottle\", \"calculator\", \"desk_chair\", \"desk_lamp\", \"desktop_computer\", \"file_cabinet\", \"unk\"]\n",
    "class_list = [\"bicycle\", \"bus\", \"car\", \"motorcycle\", \"train\", \"truck\", \"unk\"]\n",
    "# ['airplane', 'bicycle', 'bus', 'car', 'horse', 'knife', 'motorcycle',\n",
    "# 'person', 'plant', 'skateboard', 'train', 'truck', 'unk']\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GradReverse(Function):\n",
    "    def __init__(self, lambd):\n",
    "        self.lambd = lambd\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x.view_as(x)\n",
    "\n",
    "    def backward(self, grad_output):\n",
    "        return (grad_output * -self.lambd)\n",
    "\n",
    "def grad_reverse(x, lambd=1.0):\n",
    "    return GradReverse(lambd)(x)\n",
    "\n",
    "class ResNetBase(nn.Module):\n",
    "    def __init__(self, option='resnet18', pret=True, unit_size=100):\n",
    "        super(ResNetBase, self).__init__()\n",
    "        self.dim = 2048\n",
    "        if option == 'resnet18':\n",
    "            model_ft = models.resnet18(pretrained=pret)\n",
    "            self.dim = 512\n",
    "        if option == 'resnet50':\n",
    "            model_ft = models.resnet50(pretrained=pret)\n",
    "        if option == 'resnet101':\n",
    "            model_ft = models.resnet101(pretrained=pret)\n",
    "        if option == 'resnet152':\n",
    "            model_ft = models.resnet152(pretrained=pret)\n",
    "        mod = list(model_ft.children())\n",
    "        mod.pop()\n",
    "        self.features = nn.Sequential(*mod)\n",
    "        # default unit size 100\n",
    "        self.linear1 = nn.Linear(2048, unit_size)\n",
    "        self.bn1 = nn.BatchNorm1d(unit_size, affine=True)\n",
    "        self.linear2 = nn.Linear(unit_size, unit_size)\n",
    "        self.bn2 = nn.BatchNorm1d(unit_size, affine=True)\n",
    "        self.linear3 = nn.Linear(unit_size, unit_size)\n",
    "        self.bn3 = nn.BatchNorm1d(unit_size, affine=True)\n",
    "        self.linear4 = nn.Linear(unit_size, unit_size)\n",
    "        self.bn4 = nn.BatchNorm1d(unit_size, affine=True)\n",
    "    def forward(self, x,reverse=False):\n",
    "\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.size(0), self.dim)\n",
    "        # best with dropout\n",
    "        if reverse:\n",
    "            x = F.dropout(F.relu(self.bn1(self.linear1(x))))\n",
    "            x = x.detach()\n",
    "            return x\n",
    "\n",
    "        x = F.dropout(F.relu(self.bn1(self.linear1(x))), training=self.training)\n",
    "        return x\n",
    "\n",
    "\n",
    "class Classifier(nn.Module):\n",
    "    def __init__(self, num_classes=7, unit_size=1000):\n",
    "        super(Classifier, self).__init__()\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(unit_size, num_classes)\n",
    "        )\n",
    "\n",
    "    def set_lambda(self, lambd):\n",
    "        self.lambd = lambd\n",
    "\n",
    "    def forward(self, x, dropout=False, return_feat=False, reverse=False):\n",
    "        if reverse:\n",
    "            x = grad_reverse(x, self.lambd)\n",
    "        x = self.classifier(x)\n",
    "        return x\n",
    "    \n",
    "class GradientReverseLayer(torch.autograd.Function):\n",
    "    @staticmethod\n",
    "    def forward(ctx, coeff, input):\n",
    "        ctx.coeff = coeff\n",
    "        return input.view_as(input)\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_outputs):\n",
    "        coeff = ctx.coeff\n",
    "        return None, -coeff * grad_outputs\n",
    "    \n",
    "class GradientReverseModule(nn.Module):\n",
    "    def __init__(self, scheduler):\n",
    "        super(GradientReverseModule, self).__init__()\n",
    "        self.scheduler = scheduler\n",
    "        self.register_buffer('global_step', torch.zeros(1))\n",
    "        self.coeff = 0.0\n",
    "        self.grl = GradientReverseLayer.apply\n",
    "\n",
    "    def forward(self, x):\n",
    "        self.coeff = self.scheduler(self.global_step.item())\n",
    "        if self.training:\n",
    "            self.global_step += 1.0\n",
    "        return self.grl(self.coeff, x)\n",
    "    \n",
    "class LeakySoftmax(nn.Module):\n",
    "    def __init__(self, coeff=1.0, dim=-1):\n",
    "        super(LeakySoftmax, self).__init__()\n",
    "        self.softmax = torch.nn.Softmax(dim=dim)\n",
    "        self.coeff = coeff\n",
    "        self.dim = dim\n",
    "        \n",
    "    def forward(self, x):\n",
    "        shape = list(x.size())\n",
    "        shape[self.dim] = 1\n",
    "        leaky = (torch.ones(*shape, dtype=x.dtype) * np.log(self.coeff)).to(x.device)\n",
    "        concat = torch.cat([x, leaky], dim=self.dim)\n",
    "        y = self.softmax(concat)\n",
    "        prob_slicing = [slice(None, None, 1) for i in shape]\n",
    "        prob_slicing[self.dim] = slice(None, -1, 1)\n",
    "        prob = y[tuple(prob_slicing)]\n",
    "        prob_slicing[self.dim] = slice(-1, None, 1)\n",
    "        total_prob = 1.0 - y[tuple(prob_slicing)]\n",
    "        return prob, total_prob\n",
    "    \n",
    "class AugNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(AugNet, self).__init__()\n",
    "        classifier_output_dim = num_class-1\n",
    "        self.classifier_auxiliary = nn.Sequential(\n",
    "            nn.Linear(1000, 1024),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(1024,1024),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(1024, classifier_output_dim),\n",
    "            LeakySoftmax(classifier_output_dim)\n",
    "        )\n",
    "\n",
    "    def forward(self, x, k):\n",
    "        y_aug, d_aug = self.classifier_auxiliary(x)\n",
    "        d_aug = d_aug * k\n",
    "        return y_aug, d_aug\n",
    "    \n",
    "def get_optimizer_visd(lr, G, C, C_aug, update_lower=False):\n",
    "    if not update_lower:\n",
    "        params = list(list(G.linear1.parameters()) + list(G.linear2.parameters()) + list(\n",
    "            G.bn1.parameters()) + list(G.bn2.parameters())) \n",
    "    else:\n",
    "        params = G.parameters()\n",
    "    optimizer_g = opt.SGD(params, lr=lr, momentum=0.9, weight_decay=0.0005,nesterov=True)\n",
    "    optimizer_c = opt.SGD(list(C.parameters()), momentum=0.9, lr=lr,\n",
    "                          weight_decay=0.0005, nesterov=True)\n",
    "    optimizer_C_aug = opt.SGD(list(C_aug.parameters()), lr=lr,momentum=0.9, weight_decay=0.0005,nesterov=True)\n",
    "    #optimizer_des = opt.SGD(list(des.parameters()), lr=lr,momentum=0.9, weight_decay=0.0005,nesterov=True)\n",
    "    return optimizer_g, optimizer_c, optimizer_C_aug\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G = ResNetBase(args.net, unit_size=args.unit_size)\n",
    "C = Classifier(num_classes=num_class, unit_size=args.unit_size)\n",
    "classifier_aug = AugNet()\n",
    "\n",
    "#G.load_state_dict(torch.load('latestG3.pt'))\n",
    "#C.load_state_dict(torch.load('latestC3.pt'))\n",
    "#classifier_aug.load_state_dict(torch.load('latestCaug3.pt'))\n",
    "\n",
    "if args.cuda:\n",
    "    G.cuda()\n",
    "    C.cuda()\n",
    "    classifier_aug.cuda()\n",
    "    criterion = nn.CrossEntropyLoss().cuda()\n",
    "    \n",
    "opt_c, opt_g, opt_caug = get_optimizer_visd(args.lr, G, C,classifier_aug,\n",
    "                            update_lower=args.update_lower)\n",
    "\n",
    "print(args.save_path)\n",
    "best_acc = 0\n",
    "test_acc = 0\n",
    "def train(num_epoch):\n",
    "    i = 0\n",
    "    global best_acc\n",
    "    global test_acc\n",
    "    print('train start!')\n",
    "    for ep in range(num_epoch):\n",
    "        G.train()\n",
    "        C.train()\n",
    "        classifier_aug.train()\n",
    "        t = 0\n",
    "        for batch_idx, data in enumerate(dataset_train):\n",
    "            i += 1\n",
    "            if i % 1000 == 0:\n",
    "                print('iteration %d', i)\n",
    "            if args.cuda:\n",
    "                img_s = data['S']\n",
    "                label_s = data['S_label']\n",
    "                img_t = data['T']\n",
    "                img_s, label_s = Variable(img_s.cuda()), \\\n",
    "                                 Variable(label_s.cuda())\n",
    "                img_t = Variable(img_t.cuda())\n",
    "                label_t = data['T_label']\n",
    "                label_t = Variable(label_t.cuda())\n",
    "            if len(img_t) < batch_size:\n",
    "                break\n",
    "            if len(img_s) < batch_size:\n",
    "                break\n",
    "                \n",
    "            opt_g.zero_grad()\n",
    "            opt_c.zero_grad()\n",
    "           \n",
    "            feat = G(img_s)\n",
    "            Caug_feat = feat.detach()\n",
    "            out_s = C(feat)\n",
    "            out_S = F.softmax(out_s.detach())\n",
    "            probs1 = torch.sum(out_S[:, :num_class - 1], 1).view(-1, 1)\n",
    "            probs2 = out_S[:, num_class - 1].contiguous().view(-1, 1)\n",
    "            loss_s = criterion(out_s, label_s.long())\n",
    "            loss_s.backward()\n",
    "            opt_g.step()\n",
    "            opt_c.step()\n",
    "            \n",
    "            y_onehot = torch.zeros((batch_size, num_class-1)).cuda()\n",
    "            y_onehot.scatter_(1, label_s.long().unsqueeze(1), 1)\n",
    "            y_onehot.requires_grad_(False)\n",
    "            \n",
    "            p = 1.0\n",
    "            C.set_lambda(p)\n",
    "            feat_t = G(img_t)\n",
    "            out_t = C(feat_t)\n",
    "            out_t = F.softmax(out_t)\n",
    "            prob1 = torch.sum(out_t[:, :num_class - 1], 1).view(-1, 1)\n",
    "            prob2 = out_t[:, num_class - 1].contiguous().view(-1, 1)\n",
    "            \n",
    "            Caug_feat_t = feat_t.detach()\n",
    "            \n",
    "            opt_caug.zero_grad()\n",
    "            predictprob_source, domainprob_source = classifier_aug.forward(Caug_feat,probs1.detach())\n",
    "            predictprob_target, domainprob_target = classifier_aug.forward(Caug_feat_t,prob1.detach())\n",
    "            \n",
    "           \n",
    "            c_aug = nn.BCELoss(reduction ='none')(predictprob_source, y_onehot)\n",
    "            c_aug = torch.sum(c_aug) / label_s.numel()\n",
    "\n",
    "            adv_loss = nn.BCELoss()(domainprob_source, torch.ones_like(domainprob_source))\n",
    "            adv_loss += nn.BCELoss()(domainprob_target, torch.zeros_like(domainprob_target))\n",
    "            \n",
    "            loss_aug = 1.0 * adv_loss + 1.0 * c_aug\n",
    "            loss_aug.backward()\n",
    "            opt_caug.step()\n",
    "            \n",
    "            opt_g.zero_grad()\n",
    "            opt_c.zero_grad()\n",
    "            p = 1.0\n",
    "            C.set_lambda(p)\n",
    "            feat_t = G(img_t)\n",
    "            out_t = C(feat_t, reverse=True)\n",
    "            out_t = F.softmax(out_t)\n",
    "            prob1 = torch.sum(out_t[:, :num_class - 1], 1).view(-1, 1)\n",
    "            prob2 = out_t[:, num_class - 1].contiguous().view(-1, 1)\n",
    "            \n",
    "            predictprob_target, domainprob_target = classifier_aug.forward(feat_t.detach(),prob1.detach())\n",
    "        \n",
    "            prob = torch.cat((prob1, prob2), 1)\n",
    "            weight = (domainprob_target.detach())  \n",
    "            loss_t = bce_loss(prob, weight)\n",
    "\n",
    "            loss_t.backward()\n",
    "            \n",
    "            opt_g.step()\n",
    "            opt_c.step()\n",
    "           \n",
    "    \n",
    "            if batch_idx % args.log_interval == 0:\n",
    "                print('Train Ep: {} [{}/{} ({:.0f}%)]\\tLoss Source: {:.6f}\\t Loss Target: {:.6f}\\t Loss Aug: {:.6f}'.format(\n",
    "                    ep, batch_idx * len(data), 70000,\n",
    "                        100. * batch_idx / 70000, loss_s.data, loss_t.data, loss_aug.data))\n",
    "    \n",
    "            if ep > 0 and batch_idx % 1000 == 0:\n",
    "                test()\n",
    "                G.train()\n",
    "                C.train()\n",
    "                classifier_aug.train()\n",
    "                \n",
    "            if test_acc > best_acc:\n",
    "                best_acc = test_acc\n",
    "                print(best_acc)\n",
    "                print('saving starts')\n",
    "                torch.save(G.state_dict(),\"G.pt\")\n",
    "                torch.save(C.state_dict(),\"C.pt\")\n",
    "                torch.save(classifier_aug.state_dict(),\"Caug.pt\")\n",
    "                print('saving ends')\n",
    "\n",
    "\n",
    "def test():\n",
    "    G.eval()\n",
    "    C.eval()\n",
    "    correct = 0\n",
    "    size = 0\n",
    "    global test_acc\n",
    "    per_class_num = np.zeros((num_class))\n",
    "    per_class_correct = np.zeros((num_class)).astype(np.float32)\n",
    "    for batch_idx, data in enumerate(dataset_test):\n",
    "        if args.cuda:\n",
    "            img_t, label_t, path_t = data[0], data[1], data[2]\n",
    "            img_t, label_t = Variable(img_t.cuda(), volatile=True), \\\n",
    "                             Variable(label_t.cuda(), volatile=True)\n",
    "        feat = G(img_t)\n",
    "        out_t = C(feat)\n",
    "        #print(out_t.size())\n",
    "        pred = out_t.data.max(1)[1]\n",
    "        #print(pred)\n",
    "        k = label_t.data.size()[0]\n",
    "        correct += pred.eq(label_t.data.long()).cpu().sum()\n",
    "        pred = pred.cpu().numpy()\n",
    "        for t in range(num_class):\n",
    "            t_ind = np.where(label_t.data.long().cpu().numpy() == t)\n",
    "            correct_ind = np.where(pred[t_ind[0]] == t)\n",
    "            per_class_correct[t] += float(len(correct_ind[0]))\n",
    "            per_class_num[t] += float(len(t_ind[0]))\n",
    "        size += k\n",
    "    per_class_acc = per_class_correct / per_class_num\n",
    "    test_acc = 100. * correct / size\n",
    "\n",
    "    print(\n",
    "        '\\nTest set including unknown classes:  Accuracy: {}/{} ({:.0f}%)  ({:.4f}%)\\n'.format(\n",
    "            correct, size,\n",
    "            100. * correct / size, float(per_class_acc.mean())))\n",
    "    for ind, category in enumerate(class_list):\n",
    "        print('%s:%s' % (category, per_class_acc[ind]))\n",
    "        \n",
    "\n",
    "train(args.epochs + 1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}