Bert_Information_Retrieval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1,2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm, trange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "url=\"http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv\"\n",
    "original_data=pd.read_csv(url,error_bad_lines=False,sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(404290, 6)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "original_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "original_data = original_data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = original_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>404285</th>\n",
       "      <td>404285</td>\n",
       "      <td>433578</td>\n",
       "      <td>379845</td>\n",
       "      <td>How many keywords are there in the Racket prog...</td>\n",
       "      <td>How many keywords are there in PERL Programmin...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404286</th>\n",
       "      <td>404286</td>\n",
       "      <td>18840</td>\n",
       "      <td>155606</td>\n",
       "      <td>Do you believe there is life after death?</td>\n",
       "      <td>Is it true that there is life after death?</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404287</th>\n",
       "      <td>404287</td>\n",
       "      <td>537928</td>\n",
       "      <td>537929</td>\n",
       "      <td>What is one coin?</td>\n",
       "      <td>What's this coin?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404288</th>\n",
       "      <td>404288</td>\n",
       "      <td>537930</td>\n",
       "      <td>537931</td>\n",
       "      <td>What is the approx annual cost of living while...</td>\n",
       "      <td>I am having little hairfall problem but I want...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404289</th>\n",
       "      <td>404289</td>\n",
       "      <td>537932</td>\n",
       "      <td>537933</td>\n",
       "      <td>What is like to have sex with cousin?</td>\n",
       "      <td>What is it like to have sex with your cousin?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    qid1    qid2  \\\n",
       "404285  404285  433578  379845   \n",
       "404286  404286   18840  155606   \n",
       "404287  404287  537928  537929   \n",
       "404288  404288  537930  537931   \n",
       "404289  404289  537932  537933   \n",
       "\n",
       "                                                question1  \\\n",
       "404285  How many keywords are there in the Racket prog...   \n",
       "404286          Do you believe there is life after death?   \n",
       "404287                                  What is one coin?   \n",
       "404288  What is the approx annual cost of living while...   \n",
       "404289              What is like to have sex with cousin?   \n",
       "\n",
       "                                                question2  is_duplicate  \n",
       "404285  How many keywords are there in PERL Programmin...             0  \n",
       "404286         Is it true that there is life after death?             1  \n",
       "404287                                  What's this coin?             0  \n",
       "404288  I am having little hairfall problem but I want...             0  \n",
       "404289      What is it like to have sex with your cousin?             0  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['question1','question2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_sub = data[columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = data['is_duplicate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import pickle\n",
    "import torch.nn as nn\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertConfig\n",
    "from pytorch_pretrained_bert import BertForTokenClassification, BertAdam,BertForMaskedLM\n",
    "from pytorch_pretrained_bert.modeling import PreTrainedBertModel, BertModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tue Feb  5 23:47:28 2019       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 387.26                 Driver Version: 387.26                    |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  GeForce GTX 108...  Off  | 00000000:06:00.0 Off |                  N/A |\r\n",
      "| 30%   53C    P0    63W / 250W |     10MiB / 11172MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   1  GeForce GTX 108...  Off  | 00000000:09:00.0 Off |                  N/A |\r\n",
      "| 27%   49C    P0    62W / 250W |     10MiB / 11172MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   2  GeForce GTX 108...  Off  | 00000000:0A:00.0 Off |                  N/A |\r\n",
      "| 23%   41C    P0    61W / 250W |     10MiB / 11172MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                       GPU Memory |\r\n",
      "|  GPU       PID   Type   Process name                             Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|  No running processes found                                                 |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for index,row in data_sub.iterrows():\n",
    "    for column in columns:\n",
    "        data_sub.at[index,column] = tokenizer.tokenize(row[column])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>404285</th>\n",
       "      <td>[how, many, key, ##words, are, there, in, the,...</td>\n",
       "      <td>[how, many, key, ##words, are, there, in, per,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404286</th>\n",
       "      <td>[do, you, believe, there, is, life, after, dea...</td>\n",
       "      <td>[is, it, true, that, there, is, life, after, d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404287</th>\n",
       "      <td>[what, is, one, coin, ?]</td>\n",
       "      <td>[what, ', s, this, coin, ?]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404288</th>\n",
       "      <td>[what, is, the, approx, annual, cost, of, livi...</td>\n",
       "      <td>[i, am, having, little, hair, ##fall, problem,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404289</th>\n",
       "      <td>[what, is, like, to, have, sex, with, cousin, ?]</td>\n",
       "      <td>[what, is, it, like, to, have, sex, with, your...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question1  \\\n",
       "404285  [how, many, key, ##words, are, there, in, the,...   \n",
       "404286  [do, you, believe, there, is, life, after, dea...   \n",
       "404287                           [what, is, one, coin, ?]   \n",
       "404288  [what, is, the, approx, annual, cost, of, livi...   \n",
       "404289   [what, is, like, to, have, sex, with, cousin, ?]   \n",
       "\n",
       "                                                question2  \n",
       "404285  [how, many, key, ##words, are, there, in, per,...  \n",
       "404286  [is, it, true, that, there, is, life, after, d...  \n",
       "404287                        [what, ', s, this, coin, ?]  \n",
       "404288  [i, am, having, little, hair, ##fall, problem,...  \n",
       "404289  [what, is, it, like, to, have, sex, with, your...  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sub.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data_bert_preprocessed.pickle', 'wb') as sub_data:\n",
    "    pickle.dump(data_sub, sub_data, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data_bert_preprocessed.pickle', 'rb') as handle:\n",
    "    data_sub = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = max(data_sub.question1.map(lambda x: len(x)).max(),data_sub.question1.map(lambda x: len(x)).max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "153"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MAX_LEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index,row in data_sub.iterrows():\n",
    "    for column in columns:\n",
    "        data_sub.at[index,column] = pad_sequences([tokenizer.convert_tokens_to_ids(row[column])],\n",
    "                          maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>404285</th>\n",
       "      <td>[[2129, 2116, 3145, 22104, 2024, 2045, 1999, 1...</td>\n",
       "      <td>[[2129, 2116, 3145, 22104, 2024, 2045, 1999, 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404286</th>\n",
       "      <td>[[2079, 2017, 2903, 2045, 2003, 2166, 2044, 23...</td>\n",
       "      <td>[[2003, 2009, 2995, 2008, 2045, 2003, 2166, 20...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404287</th>\n",
       "      <td>[[2054, 2003, 2028, 9226, 1029, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>[[2054, 1005, 1055, 2023, 9226, 1029, 0, 0, 0,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404288</th>\n",
       "      <td>[[2054, 2003, 1996, 22480, 3296, 3465, 1997, 2...</td>\n",
       "      <td>[[1045, 2572, 2383, 2210, 2606, 13976, 3291, 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404289</th>\n",
       "      <td>[[2054, 2003, 2066, 2000, 2031, 3348, 2007, 55...</td>\n",
       "      <td>[[2054, 2003, 2009, 2066, 2000, 2031, 3348, 20...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                question1  \\\n",
       "404285  [[2129, 2116, 3145, 22104, 2024, 2045, 1999, 1...   \n",
       "404286  [[2079, 2017, 2903, 2045, 2003, 2166, 2044, 23...   \n",
       "404287  [[2054, 2003, 2028, 9226, 1029, 0, 0, 0, 0, 0,...   \n",
       "404288  [[2054, 2003, 1996, 22480, 3296, 3465, 1997, 2...   \n",
       "404289  [[2054, 2003, 2066, 2000, 2031, 3348, 2007, 55...   \n",
       "\n",
       "                                                question2  \n",
       "404285  [[2129, 2116, 3145, 22104, 2024, 2045, 1999, 2...  \n",
       "404286  [[2003, 2009, 2995, 2008, 2045, 2003, 2166, 20...  \n",
       "404287  [[2054, 1005, 1055, 2023, 9226, 1029, 0, 0, 0,...  \n",
       "404288  [[1045, 2572, 2383, 2210, 2606, 13976, 3291, 2...  \n",
       "404289  [[2054, 2003, 2009, 2066, 2000, 2031, 3348, 20...  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sub.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data_bert_integer.pickle', 'wb') as sub_data:\n",
    "    pickle.dump(data_sub, sub_data, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data_bert_integer.pickle', 'rb') as handle:\n",
    "    data_sub = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#attention_masks1 = []\n",
    "#attention_masks2 = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for index,row in data_sub.iterrows():\n",
    "#            attention_masks1.append([float(i>0) for i in ii] for ii in row['question1'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for index,row in data_sub.iterrows():\n",
    "#            attention_masks2.append([float(i>0) for i in ii] for ii in row['question2'] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#attention_masks = [[float(i>0) for i in ii] for ii in input_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs, val_inputs, tr_tags, val_tags = train_test_split(data_sub[:50000], labels[:50000], \n",
    "                                                            random_state=2018, test_size=0.1)\n",
    "#tr_masks1, val_masks1,tr_masks2, val_masks2 = train_test_split(attention_masks1, attention_masks2,\n",
    "#                                             random_state=2018, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs1 = tr_inputs.question1\n",
    "tr_inputs2 = tr_inputs.question2\n",
    "val_inputs1 = val_inputs.question1\n",
    "val_inputs2 = val_inputs.question2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "tr_inputs1 = list(itertools.chain(*tr_inputs1))\n",
    "tr_inputs2 = list(itertools.chain(*tr_inputs2))\n",
    "val_inputs1 = list(itertools.chain(*val_inputs1))\n",
    "val_inputs2 = list(itertools.chain(*val_inputs2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_tags = np.array(tr_tags)\n",
    "val_tags = np.array(val_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs1 = torch.tensor(tr_inputs1)\n",
    "tr_inputs2 = torch.tensor(tr_inputs2)\n",
    "val_inputs1 = torch.tensor(val_inputs1)\n",
    "val_inputs2 = torch.tensor(val_inputs2)\n",
    "tr_labels = torch.tensor(tr_tags)\n",
    "val_labels = torch.tensor(val_tags)\n",
    "#tr_masks1 = torch.tensor(tr_masks1)\n",
    "#val_masks1 = torch.tensor(val_masks1)\n",
    "#tr_masks2 = torch.tensor(tr_masks2)\n",
    "#val_masks2 = torch.tensor(val_masks2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([45000])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tr_labels.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = TensorDataset(tr_inputs1, tr_inputs2, tr_labels)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=16)\n",
    "\n",
    "valid_data = TensorDataset(val_inputs1, val_inputs2, val_labels)\n",
    "valid_sampler = SequentialSampler(valid_data)\n",
    "valid_dataloader = DataLoader(valid_data, sampler=valid_sampler, batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BertForQuoraClassification(PreTrainedBertModel):\n",
    "\n",
    "    def __init__(self, config, num_labels=1):\n",
    "        super(BertForQuoraClassification, self).__init__(config)\n",
    "        self.num_labels = num_labels\n",
    "        self.bert = BertModel(config)\n",
    "        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n",
    "        #self.classifier = nn.Linear(config.hidden_size, num_labels)\n",
    "        self.apply(self.init_bert_weights)\n",
    "\n",
    "    def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None):\n",
    "        _, pooled_output = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        #logits = self.classifier(pooled_output)\n",
    "        return pooled_output\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert = BertForQuoraClassification.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyEnsemble(nn.Module):\n",
    "    def __init__(self, bert):\n",
    "        super(MyEnsemble, self).__init__()\n",
    "        self.bert = bert\n",
    "        self.classifier = nn.Linear(768*2, 512)\n",
    "        self.fc1 = nn.Linear(512,256)\n",
    "        self.fc2 = nn.Linear(256,1)\n",
    "        \n",
    "    def forward(self, x1, x2):\n",
    "        x1 = self.bert(x1)\n",
    "        x2 = self.bert(x2)\n",
    "        x = torch.cat((x1, x2), dim=1)\n",
    "        x = self.classifier(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.fc1(x)\n",
    "        x = F.relu(x)\n",
    "        x = self.fc2(x)\n",
    "        return F.sigmoid(x)\n",
    "\n",
    "\n",
    "final = MyEnsemble(bert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Let's use 3 GPUs!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DataParallel(\n",
       "  (module): MyEnsemble(\n",
       "    (bert): BertForQuoraClassification(\n",
       "      (bert): BertModel(\n",
       "        (embeddings): BertEmbeddings(\n",
       "          (word_embeddings): Embedding(30522, 768)\n",
       "          (position_embeddings): Embedding(512, 768)\n",
       "          (token_type_embeddings): Embedding(2, 768)\n",
       "          (LayerNorm): BertLayerNorm()\n",
       "          (dropout): Dropout(p=0.1)\n",
       "        )\n",
       "        (encoder): BertEncoder(\n",
       "          (layer): ModuleList(\n",
       "            (0): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (1): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (2): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (3): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (4): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (5): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (6): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (7): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (8): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (9): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (10): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "            (11): BertLayer(\n",
       "              (attention): BertAttention(\n",
       "                (self): BertSelfAttention(\n",
       "                  (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "                (output): BertSelfOutput(\n",
       "                  (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                  (LayerNorm): BertLayerNorm()\n",
       "                  (dropout): Dropout(p=0.1)\n",
       "                )\n",
       "              )\n",
       "              (intermediate): BertIntermediate(\n",
       "                (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              )\n",
       "              (output): BertOutput(\n",
       "                (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "                (LayerNorm): BertLayerNorm()\n",
       "                (dropout): Dropout(p=0.1)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (pooler): BertPooler(\n",
       "          (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "          (activation): Tanh()\n",
       "        )\n",
       "      )\n",
       "      (dropout): Dropout(p=0.1)\n",
       "    )\n",
       "    (classifier): Linear(in_features=1536, out_features=512, bias=True)\n",
       "    (fc1): Linear(in_features=512, out_features=256, bias=True)\n",
       "    (fc2): Linear(in_features=256, out_features=1, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if torch.cuda.device_count() > 1:\n",
    "    print(\"Let's use\", torch.cuda.device_count(), \"GPUs!\")\n",
    "    model = nn.DataParallel(final,device_ids=(0,1,2))\n",
    "\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "FULL_FINETUNING = True\n",
    "if FULL_FINETUNING:\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.01},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.0}\n",
    "    ]\n",
    "else:\n",
    "    param_optimizer = list(model.classifier.named_parameters()) \n",
    "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
    "optimizer = BertAdam(optimizer_grouped_parameters, lr=3e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from seqeval.metrics import f1_score\n",
    "\n",
    "def flat_accuracy(preds, labels):\n",
    "    #pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    #acc = (np.abs(preds - labels_flat))/len(labels_flat)\n",
    "    super_threshold_indices = preds > 0.5\n",
    "    preds[super_threshold_indices] = 1\n",
    "    super_threshold_indices = preds < 0.5\n",
    "    preds[super_threshold_indices] = 0\n",
    "    return np.sum(preds == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wed Feb  6 00:11:01 2019       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 387.26                 Driver Version: 387.26                    |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  GeForce GTX 108...  Off  | 00000000:06:00.0 Off |                  N/A |\r\n",
      "| 35%   60C    P5    17W / 250W |   6923MiB / 11172MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   1  GeForce GTX 108...  Off  | 00000000:09:00.0 Off |                  N/A |\r\n",
      "| 31%   49C    P8    11W / 250W |   4869MiB / 11172MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "|   2  GeForce GTX 108...  Off  | 00000000:0A:00.0 Off |                  N/A |\r\n",
      "| 23%   36C    P8     9W / 250W |   3887MiB / 11172MiB |      0%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                       GPU Memory |\r\n",
      "|  GPU       PID   Type   Process name                             Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|    0     12290      C   ...esolution-master_new/SR_new/bin/python3  6913MiB |\r\n",
      "|    1     12290      C   ...esolution-master_new/SR_new/bin/python3  4859MiB |\r\n",
      "|    2     12290      C   ...esolution-master_new/SR_new/bin/python3  3877MiB |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:   0%|          | 0/5 [00:00<?, ?it/s]/users/aditya.sharma/super-resolution-master_new/SR_new/lib/python3.5/site-packages/torch/nn/functional.py:1006: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.588601137342957\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  20%|██        | 1/5 [2:21:04<9:24:18, 8464.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss: 0.5824119295641969\n",
      "Validation Accuracy: 18.469872048442063\n",
      "Train loss: 0.6460132952122667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  40%|████      | 2/5 [4:41:35<7:02:43, 8454.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss: 0.6592472347466252\n",
      "Validation Accuracy: 20.17009493670886\n",
      "Train loss: 0.6596534408949346\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  60%|██████    | 3/5 [7:02:28<4:41:48, 8454.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss: 0.659038046162717\n",
      "Validation Accuracy: 20.17009493670886\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-f566f164dc4e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinary_cross_entropy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb_labels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m         \u001b[0mtr_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m         \u001b[0mnb_tr_examples\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mb_input_ids1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0mnb_tr_steps\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "max_grad_norm = 1.0\n",
    "epoch = 5\n",
    "for i in trange(epoch,desc='Epoch'):  \n",
    "    model.train()\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    for batch_idx, batch in enumerate(train_dataloader):\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids1,b_input_ids2, b_labels = batch\n",
    "        optimizer.zero_grad()\n",
    "        output = model(b_input_ids1,b_input_ids2)\n",
    "        #loss_fct = CrossEntropyLoss()\n",
    "        #loss = loss_fct(output.view(-1, 1), b_labels.view(-1))\n",
    "        loss = F.binary_cross_entropy(output.view(-1), b_labels.float().view(-1))\n",
    "        loss.backward()\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += b_input_ids1.size(0)\n",
    "        nb_tr_steps += 1\n",
    "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        model.zero_grad()\n",
    "        \n",
    "    print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
    "    # VALIDATION on validation set\n",
    "    model.eval()\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    predictions , true_labels = [], []\n",
    "    for batch in valid_dataloader:\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids1,b_input_ids2, val_labels = batch\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            logits = model(b_input_ids1,b_input_ids2)\n",
    "            tmp_eval_loss = loss = F.binary_cross_entropy(logits.view(-1), val_labels.float().view(-1))\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = val_labels.to('cpu').numpy()\n",
    "        #predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n",
    "        #predictions.append(logits)\n",
    "        true_labels.append(label_ids)\n",
    "        #preds = np.array(logits)\n",
    "        #super_threshold_indices = preds > 0.5\n",
    "        #preds[super_threshold_indices] = 1\n",
    "        #super_threshold_indices = preds <= 0.5\n",
    "        #preds[super_threshold_indices] = 0\n",
    "            \n",
    "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "        \n",
    "        eval_loss += tmp_eval_loss.mean().item()\n",
    "        eval_accuracy += tmp_eval_accuracy\n",
    "        \n",
    "        nb_eval_examples += b_input_ids1.size(0)\n",
    "        nb_eval_steps += 1\n",
    "    eval_loss = eval_loss/nb_eval_steps\n",
    "    print(\"Validation loss: {}\".format(eval_loss))\n",
    "    print(\"Validation Accuracy: {}\".format(eval_accuracy/nb_eval_steps))\n",
    "    #pred_tags = [for p in predictions for p_i in p]\n",
    "    #valid_tags = [l in true_labels for l_i in l for l_ii in l_i]\n",
    "    #print(\"F1-Score: {}\".format(f1_score(preds, label_ids)))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}