Pytorch/(Pytorch) (3) BERT-large to Classify Twitter NLP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0128 18:50:40.863723  8744 file_utils.py:35] PyTorch version 1.3.1 available.\n",
      "I0128 18:50:44.803600  8744 file_utils.py:48] TensorFlow version 2.0.0 available.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import string\n",
    "import os\n",
    "import torch\n",
    "import numpy as np\n",
    "import tqdm\n",
    "from transformers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_dataset = 'C:\\\\Users\\\\bokhy\\\\Python\\\\pytorch\\\\pytorch\\\\data\\\\nlp-getting-started\\\\'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(os.path.join(path_to_dataset, 'train.csv'))\n",
    "test = pd.read_csv(os.path.join(path_to_dataset, 'test.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text  \\\n",
       "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n",
       "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n",
       "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n",
       "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n",
       "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n",
       "\n",
       "   target  \n",
       "0       1  \n",
       "1       1  \n",
       "2       1  \n",
       "3       1  \n",
       "4       1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just happened a terrible car crash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Heard about #earthquake is different cities, s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>there is a forest fire at spot pond, geese are...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text\n",
       "0   0     NaN      NaN                 Just happened a terrible car crash\n",
       "1   2     NaN      NaN  Heard about #earthquake is different cities, s...\n",
       "2   3     NaN      NaN  there is a forest fire at spot pond, geese are...\n",
       "3   9     NaN      NaN           Apocalypse lighting. #Spokane #wildfires\n",
       "4  11     NaN      NaN      Typhoon Soudelor kills 28 in China and Taiwan"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0127 22:15:40.058504 24624 file_utils.py:362] https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt not found in cache or force_download set to True, downloading to C:\\Users\\bokhy\\AppData\\Local\\Temp\\tmpr9i9gos6\n",
      "I0127 22:15:40.768607 24624 file_utils.py:377] copying C:\\Users\\bokhy\\AppData\\Local\\Temp\\tmpr9i9gos6 to cache at C:\\Users\\bokhy\\.cache\\torch\\transformers\\9b3c03a36e83b13d5ba95ac965c9f9074a99e14340c523ab405703179e79fc46.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "I0127 22:15:40.771598 24624 file_utils.py:381] creating metadata file for C:\\Users\\bokhy\\.cache\\torch\\transformers\\9b3c03a36e83b13d5ba95ac965c9f9074a99e14340c523ab405703179e79fc46.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "I0127 22:15:40.773594 24624 file_utils.py:390] removing temp file C:\\Users\\bokhy\\AppData\\Local\\Temp\\tmpr9i9gos6\n",
      "I0127 22:15:40.774590 24624 tokenization_utils.py:398] loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt from cache at C:\\Users\\bokhy\\.cache\\torch\\transformers\\9b3c03a36e83b13d5ba95ac965c9f9074a99e14340c523ab405703179e79fc46.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-large-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(torch.nn.Module):\n",
    "    \n",
    "    def __init__(self, ):\n",
    "        \n",
    "        super(Model, self).__init__()\n",
    "        self.base_model = BertModel.from_pretrained('bert-base-uncased') # use pre-trained BERT model by HuggingFace\n",
    "        self.fc1 = torch.nn.Linear(768, 1) # simple logistic regression above the bert model\n",
    "        \n",
    "    def forward(self, ids, masks):\n",
    "        \n",
    "        x = self.base_model(ids, attention_mask=masks)[1]\n",
    "        x = self.fc1(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0127 22:15:44.412093 24624 configuration_utils.py:185] loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json from cache at C:\\Users\\bokhy\\.cache\\torch\\transformers\\4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.bf3b9ea126d8c0001ee8a1e8b92229871d06d36d8808208cc2449280da87785c\n",
      "I0127 22:15:44.414086 24624 configuration_utils.py:199] Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 2,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pruned_heads\": {},\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "I0127 22:15:44.788405 24624 modeling_utils.py:406] loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-pytorch_model.bin from cache at C:\\Users\\bokhy\\.cache\\torch\\transformers\\aa1ef1aede4482d0dbcd4d52baad8ae300e60902e88fcb0bebdec09afd232066.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157\n"
     ]
    }
   ],
   "source": [
    "model = Model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set-up GPU\n",
    "device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bert_encode(text, max_len=128):\n",
    "    \n",
    "    text = tokenizer.tokenize(text)\n",
    "    text = text[:max_len-2]\n",
    "    input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n",
    "    tokens = tokenizer.convert_tokens_to_ids(input_sequence)\n",
    "    tokens += [0] * (max_len - len(input_sequence))\n",
    "    pad_masks = [1] * len(input_sequence) + [0] * (max_len - len(input_sequence))\n",
    "\n",
    "    return tokens, pad_masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r'[^a-z0-9] \\n', '', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use first 6000 for training, rest for validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_text = train.text[:6000]\n",
    "val_text = train.text[6000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_text = train_text.apply(clean_text)\n",
    "val_text = val_text.apply(clean_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train token\n",
    "train_tokens = []\n",
    "train_pad_masks = []\n",
    "for text in train_text:\n",
    "    tokens, masks = bert_encode(text)\n",
    "    train_tokens.append(tokens)\n",
    "    train_pad_masks.append(masks)\n",
    "    \n",
    "train_tokens = np.array(train_tokens)\n",
    "train_pad_masks = np.array(train_pad_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Validation token\n",
    "val_tokens = []\n",
    "val_pad_masks = []\n",
    "for text in val_text:\n",
    "    tokens, masks = bert_encode(text)\n",
    "    val_tokens.append(tokens)\n",
    "    val_pad_masks.append(masks)\n",
    "    \n",
    "val_tokens = np.array(val_tokens)\n",
    "val_pad_masks = np.array(val_pad_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset(torch.utils.data.Dataset):\n",
    "    \n",
    "    def __init__(self, train_tokens, train_pad_masks, targets):\n",
    "        \n",
    "        super(Dataset, self).__init__()\n",
    "        self.train_tokens = train_tokens\n",
    "        self.train_pad_masks = train_pad_masks\n",
    "        self.targets = targets\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        \n",
    "        tokens = self.train_tokens[index]\n",
    "        masks = self.train_pad_masks[index]\n",
    "        target = self.targets[index]\n",
    "        \n",
    "        return (tokens, masks), target\n",
    "    \n",
    "    def __len__(self,):\n",
    "        \n",
    "        return len(self.train_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Accuracy Metric\n",
    "def accuracy(y_actual, y_pred):\n",
    "    y_ = y_pred > 0\n",
    "    return np.sum(y_actual == y_).astype('int') / y_actual.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = Dataset(\n",
    "                    train_tokens=train_tokens,\n",
    "                    train_pad_masks=train_pad_masks,\n",
    "                    targets=train.target[:6000]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=6, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dataset = Dataset(\n",
    "                    train_tokens=val_tokens,\n",
    "                    train_pad_masks=val_pad_masks,\n",
    "                    targets=train.target[6000:].reset_index(drop=True)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dataloader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=3, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.BCEWithLogitsLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimizaer: Use ADAM with Learning Rate 0.00001\n",
    "opt = torch.optim.Adam(model.parameters(), lr=0.00001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "99.81% valid_loss: 0.03, accuracy 1.003\n",
      "Average accuracy:  0.8314745972738545\n",
      "99.81% valid_loss: 0.01, accuracy 1.007\n",
      "Average accuracy:  0.829615861214375\n",
      "99.81% valid_loss: 0.01, accuracy 1.005\n",
      "Average accuracy:  0.8389095415117728\n"
     ]
    }
   ],
   "source": [
    "# Train for 3 Epoch for Training/Validation set\n",
    "EPOCHS = 3\n",
    "model.train()\n",
    "y_preds = []\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "        for i, ((tokens, masks), target) in enumerate(train_dataloader):\n",
    "\n",
    "            y_pred = model(                            # This is the forward pass (just put tokens and maskds inside model)\n",
    "                        tokens.long().to(device), \n",
    "                        masks.long().to(device)\n",
    "                    )\n",
    "            loss = criterion(y_pred, target[:, None].float().to(device))\n",
    "            opt.zero_grad()\n",
    "            loss.backward()\n",
    "            opt.step()\n",
    "            print('\\rEpoch: %d/%d, %f%% train_loss: %0.2f'% (epoch+1, EPOCHS, i/len(train_dataloader)*100, loss.item()), end='') \n",
    "        else: \n",
    "            avg_acc = 0\n",
    "            with torch.no_grad(): # no trainig for valid set ==> no need gradient (save memory)\n",
    "                model.eval() # Turn-off drop-out to make it as inference mode\n",
    "                for i, ((tokens, masks), target) in enumerate(val_dataloader):\n",
    "\n",
    "                    y_pred = model(\n",
    "                                tokens.long().to(device), \n",
    "                                masks.long().to(device), \n",
    "                            )\n",
    "                    valid_loss = criterion(y_pred,  target[:, None].float().to(device))\n",
    "                    acc = accuracy(target.cpu().numpy(), y_pred.detach().cpu().numpy().squeeze())\n",
    "                    avg_acc += acc\n",
    "            \n",
    "            model.train()\n",
    "               \n",
    "            print('\\r%0.2f%% valid_loss: %0.2f, accuracy %0.2f'% (i/len(val_dataloader)*100, valid_loss.item(), acc), end='')\n",
    "            print('\\nAverage accuracy: ', avg_acc / len(val_dataloader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InLineBackend.figure_format = 'retina'\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1ed1a345588>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGAJJREFUeJzt3X1wldXd7vHvr+GtApIA8aggBaqnmoSQxG2kAyWglEIdQRErKBWtLaPVtlPGGfOorRLbGUQepXg4VXoq4yg1pTLWHKsytk2lTk+RhGIwWJ4ExBLD0SBCRajOht/zRzbpJmzInWQnAdf1mdmz75d1r71WMnNl5X5Z29wdEREJw+d6ugEiItJ9FPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgGJFPpmNs3MtplZvZmVpti/0My2mlmNmf3BzL6QtO+wmW1OvCrS2XgREWkfa+s+fTPLAP4L+CrQAGwE5rr71qQyk4EN7n7QzG4DJrn7dYl9B9x9QFd1QEREoosy0i8G6t19h7t/CpQDM5MLuHulux9MrP4VGJ7eZoqISDr0ilBmGLArab0BuPQk5W8BXkpa72dmVUAcWOzuv219gJktABYA9O/f/+ILL7wwQrNEROSo6urqPe6e3Va5KKFvKbalPCdkZvOAGFCStHmEuzea2Wjgj2a2xd23H1OZ+0pgJUAsFvOqqqoIzRIRkaPM7J0o5aKc3mkAzktaHw40pvjAKcA9wAx3/+TodndvTLzvAP4EFEZpmIiIpF+U0N8IXGBmo8ysDzAHOOYuHDMrBB6nOfDfT9qeZWZ9E8tDgfHAVkREpEe0eXrH3eNmdgewDsgAnnD3WjMrA6rcvQJ4CBgA/MbMAP7h7jOAi4DHzewIzX9gFiff9SMiIt2rzVs2u5vO6YuItJ+ZVbt7rK1yeiJXRCQgCn0RkYAo9EVEAqLQFxEJiEJfRHrUBx98QEFBAQUFBZx99tkMGzasZf3TTz+NVMfNN9/Mtm3bTlpmxYoVrF69Oh1NZsKECWzevDktdXW3KE/kioh0mSFDhrQE6P3338+AAQO48847jynj7rg7n/tc6nHqqlWr2vyc22+/vfON/QzQSF9ETkn19fXk5eVx6623UlRUxO7du1mwYAGxWIzc3FzKyspayh4decfjcTIzMyktLWXs2LF8+ctf5v33m58Xvffee1m2bFlL+dLSUoqLi/nSl77EX/7yFwA+/vhjrrnmGsaOHcvcuXOJxWJtjuiffvppxowZQ15eHnfffTcA8Xicb37zmy3bly9fDsAjjzxCTk4OY8eOZd68eWn/mUWhkb6ItFj0f2vZ2vjPtNaZc+6Z3HdlboeO3bp1K6tWreKxxx4DYPHixQwePJh4PM7kyZOZPXs2OTk5xxyzf/9+SkpKWLx4MQsXLuSJJ56gtPS4rwHB3Xn99depqKigrKyMl19+mUcffZSzzz6btWvX8sYbb1BUVHTS9jU0NHDvvfdSVVXFoEGDmDJlCi+88ALZ2dns2bOHLVu2ALBv3z4AlixZwjvvvEOfPn1atnU3jfRF5JT1xS9+kUsuuaRl/ZlnnqGoqIiioiLeeusttm49/gH/z3/+80yfPh2Aiy++mJ07d6ase9asWceVee2115gzZw4AY8eOJTf35H+sNmzYwGWXXcbQoUPp3bs3119/PevXr+f8889n27Zt/OAHP2DdunUMGjQIgNzcXObNm8fq1avp3bt3u34W6aKRvoi06OiIvKv079+/Zbmuro6f/exnvP7662RmZjJv3jz+9a9/HXdMnz59WpYzMjKIx+Mp6+7bt+9xZdo7Q8GJyg8ZMoSamhpeeuklli9fztq1a1m5ciXr1q3j1Vdf5fnnn+cnP/kJb775JhkZGe36zM7SSF9ETgv//Oc/GThwIGeeeSa7d+9m3bp1af+MCRMmsGbNGgC2bNmS8j+JZOPGjaOyspIPPviAeDxOeXk5JSUlNDU14e5ce+21LFq0iE2bNnH48GEaGhq47LLLeOihh2hqauLgwYMnrb8raKQvIqeFoqIicnJyyMvLY/To0YwfPz7tn/G9732PG2+8kfz8fIqKisjLy2s5NZPK8OHDKSsrY9KkSbg7V155JVdccQWbNm3illtuwd0xMx588EHi8TjXX389H330EUeOHOGuu+5i4MCBae9DWzThmohIQjweJx6P069fP+rq6pg6dSp1dXX06nXqj4+jTrh26vdERKSbHDhwgMsvv5x4PI678/jjj58Wgd8en63eiIh0QmZmJtXV1T3djC6lC7kiIgFR6IuIBEShLyISEIW+iEhAFPoi0qMmTZp03INWy5Yt47vf/e5JjxswYAAAjY2NzJ49+4R1t3UL+LJly455SOrrX/96WubFuf/++1m6dGmn60k3hb6I9Ki5c+dSXl5+zLby8nLmzp0b6fhzzz2XZ599tsOf3zr0X3zxRTIzMztc36lOoS8iPWr27Nm88MILfPLJJwDs3LmTxsZGJkyY0HLffFFREWPGjOH5558/7vidO3eSl5cHwKFDh5gzZw75+flcd911HDp0qKXcbbfd1jIt83333QfA8uXLaWxsZPLkyUyePBmAkSNHsmfPHgAefvhh8vLyyMvLa5mWeefOnVx00UV85zvfITc3l6lTpx7zOals3ryZcePGkZ+fz9VXX82HH37Y8vk5OTnk5+e3TPT26quvtnyJTGFhIR999FGHf7ap6D59Efm3l0rh/29Jb51nj4Hpi0+4e8iQIRQXF/Pyyy8zc+ZMysvLue666zAz+vXrx3PPPceZZ57Jnj17GDduHDNmzMDMUtb185//nDPOOIOamhpqamqOmRr5pz/9KYMHD+bw4cNcfvnl1NTU8P3vf5+HH36YyspKhg4dekxd1dXVrFq1ig0bNuDuXHrppZSUlJCVlUVdXR3PPPMMv/jFL/jGN77B2rVrTzo//o033sijjz5KSUkJP/7xj1m0aBHLli1j8eLFvP322/Tt27fllNLSpUtZsWIF48eP58CBA/Tr1689P+02aaQvIj0u+RRP8qkdd+fuu+8mPz+fKVOm8O677/Lee++dsJ7169e3hG9+fj75+fkt+9asWUNRURGFhYXU1ta2OZnaa6+9xtVXX03//v0ZMGAAs2bN4s9//jMAo0aNoqCgADj59M3QPL//vn37KCkpAWD+/PmsX7++pY033HADTz/9dMuTv+PHj2fhwoUsX76cffv2pf2JYI30ReTfTjIi70pXXXUVCxcuZNOmTRw6dKhlhL569Wqampqorq6md+/ejBw5MuV0yslS/Rfw9ttvs3TpUjZu3EhWVhY33XRTm/WcbF6yo9MyQ/PUzG2d3jmR3/3ud6xfv56KigoeeOABamtrKS0t5YorruDFF19k3Lhx/P73v+fCCy/sUP2paKQvIj1uwIABTJo0iW9961vHXMDdv38/Z511Fr1796ayspJ33nnnpPVMnDix5cvP33zzTWpqaoDmaZn79+/PoEGDeO+993jppZdajhk4cGDK8+YTJ07kt7/9LQcPHuTjjz/mueee4ytf+Uq7+zZo0CCysrJa/kt46qmnKCkp4ciRI+zatYvJkyezZMkS9u3bx4EDB9i+fTtjxozhrrvuIhaL8fe//73dn3kyGumLyClh7ty5zJo165g7eW644QauvPJKYrEYBQUFbY54b7vtNm6++Wby8/MpKCiguLgYaP4WrMLCQnJzc4+blnnBggVMnz6dc845h8rKypbtRUVF3HTTTS11fPvb36awsPCkp3JO5Mknn+TWW2/l4MGDjB49mlWrVnH48GHmzZvH/v37cXd++MMfkpmZyY9+9CMqKyvJyMggJyen5VvA0kVTK4uIfAZEnVpZp3dERAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCUik0DezaWa2zczqzaw0xf6FZrbVzGrM7A9m9oWkffPNrC7xmp/OxouISPu0GfpmlgGsAKYDOcBcM8tpVexvQMzd84FngSWJYwcD9wGXAsXAfWaWlb7mi4hIe0QZ6RcD9e6+w90/BcqBmckF3L3S3Y9OSP1XYHhi+WvAK+6+190/BF4BpqWn6SIi0l5RQn8YsCtpvSGx7URuAY5ObNHeY0VEpAtFmXsn1cTVKeduMLN5QAwoac+xZrYAWAAwYsSICE0SEZGOiDLSbwDOS1ofDjS2LmRmU4B7gBnu/kl7jnX3le4ec/dYdnZ21LaLiEg7RQn9jcAFZjbKzPoAc4CK5AJmVgg8TnPgv5+0ax0w1cyyEhdwpya2iYhID2jz9I67x83sDprDOgN4wt1rzawMqHL3CuAhYADwm8QXGPzD3We4+14ze4DmPxwAZe6+t0t6IiIibdLUyiIinwGaWllERI6j0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCYhCX0QkIAp9EZGAKPRFRAKi0BcRCUik0DezaWa2zczqzaw0xf6JZrbJzOJmNrvVvsNmtjnxqkhXw0VEpP16tVXAzDKAFcBXgQZgo5lVuPvWpGL/AG4C7kxRxSF3L0hDW0VEpJPaDH2gGKh39x0AZlYOzARaQt/ddyb2HemCNoqISJpEOb0zDNiVtN6Q2BZVPzOrMrO/mtlV7WqdiIikVZSRvqXY5u34jBHu3mhmo4E/mtkWd99+zAeYLQAWAIwYMaIdVYuISHtEGek3AOclrQ8HGqN+gLs3Jt53AH8CClOUWenuMXePZWdnR61aRETaKUrobwQuMLNRZtYHmANEugvHzLLMrG9ieSgwnqRrASIi0r3aDH13jwN3AOuAt4A17l5rZmVmNgPAzC4xswbgWuBxM6tNHH4RUGVmbwCVwOJWd/2IiEg3Mvf2nJ7verFYzKuqqnq6GSIipxUzq3b3WFvl9ESuiEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBCRS6JvZNDPbZmb1ZlaaYv9EM9tkZnEzm91q33wzq0u85qer4SIi0n5thr6ZZQArgOlADjDXzHJaFfsHcBPwq1bHDgbuAy4FioH7zCyr880WEZGOiDLSLwbq3X2Hu38KlAMzkwu4+053rwGOtDr2a8Ar7r7X3T8EXgGmpaHdIiLSAVFCfxiwK2m9IbEtikjHmtkCM6sys6qmpqaIVYuISHtFCX1Lsc0j1h/pWHdf6e4xd49lZ2dHrFpERNorSug3AOclrQ8HGiPW35ljRUQkzaKE/kbgAjMbZWZ9gDlARcT61wFTzSwrcQF3amKbiIj0gDZD393jwB00h/VbwBp3rzWzMjObAWBml5hZA3At8LiZ1SaO3Qs8QPMfjo1AWWKbiIj0AHOPenq+e8RiMa+qqurpZoiInFbMrNrdY22V0xO5IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAREoS8iEhCFvohIQBT6IiIBUeiLiAQkUuib2TQz22Zm9WZWmmJ/XzP7dWL/BjMbmdg+0swOmdnmxOux9DZfRETao1dbBcwsA1gBfBVoADaaWYW7b00qdgvwobufb2ZzgAeB6xL7trt7QZrbLSIiHRBlpF8M1Lv7Dnf/FCgHZrYqMxN4MrH8LHC5mVn6mikiIukQJfSHAbuS1hsS21KWcfc4sB8Yktg3ysz+ZmavmtlXUn2AmS0wsyozq2pqampXB0REJLoooZ9qxO4Ry+wGRrh7IbAQ+JWZnXlcQfeV7h5z91h2dnaEJomISEdECf0G4Lyk9eFA44nKmFkvYBCw190/cfcPANy9GtgO/M/ONlpERDomSuhvBC4ws1Fm1geYA1S0KlMBzE8szwb+6O5uZtmJC8GY2WjgAmBHepouIiLt1ebdO+4eN7M7gHVABvCEu9eaWRlQ5e4VwC+Bp8ysHthL8x8GgIlAmZnFgcPAre6+tys6IiIibTP31qfne1YsFvOqqqqeboaIyGnFzKrdPdZWOT2RKyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAFPoiIgFR6IuIBEShLyISEIW+iEhAIoW+mU0zs21mVm9mpSn29zWzXyf2bzCzkUn7/iOxfZuZfS19TRcRkfZqM/TNLANYAUwHcoC5ZpbTqtgtwIfufj7wCPBg4tgcYA6QC0wD/neiPhER6QFRRvrFQL2773D3T4FyYGarMjOBJxPLzwKXm5kltpe7+yfu/jZQn6hPRER6QJTQHwbsSlpvSGxLWcbd48B+YEjEYzGzBWZWZWZVTU1N0VsvIiLtEiX0LcU2j1gmyrG4+0p3j7l7LDs7O0KTRESkI3pFKNMAnJe0PhxoPEGZBjPrBQwC9kY89hjV1dV7zOydCO061QwF9vR0I7qZ+hwG9fn08IUohaKE/kbgAjMbBbxL84XZ61uVqQDmA/8PmA380d3dzCqAX5nZw8C5wAXA6yf7MHc/LYf6Zlbl7rGebkd3Up/DoD5/trQZ+u4eN7M7gHVABvCEu9eaWRlQ5e4VwC+Bp8ysnuYR/pzEsbVmtgbYCsSB2939cBf1RURE2mDux51ilw74LI8MTkR9DoP6/NmiJ3LTZ2VPN6AHqM9hUJ8/QzTSFxEJiEb6IiIBUeiLiAREod8OZjbYzF4xs7rEe9YJys1PlKkzs/kp9leY2Ztd3+LO60yfzewMM/udmf3dzGrNbHH3tj66ECcV7GifzeyrZlZtZlsS75d1d9s7qjO/58T+EWZ2wMzu7K42p5276xXxBSwBShPLpcCDKcoMBnYk3rMSy1lJ+2cBvwLe7On+dHWfgTOAyYkyfYA/A9N7uk8p2p8BbAdGJ9r5BpDTqsx3gccSy3OAXyeWcxLl+wKjEvVk9HSfurjPhcC5ieU84N2e7k9X9zlp/1rgN8CdPd2fjr400m+f5InlngSuSlHma8Ar7r7X3T8EXqF5hlHMbACwEPhJN7Q1XTrcZ3c/6O6VAN48Wd8mmp/KPtWEOKlgh/vs7n9z96NP1tcC/cysb7e0unM683vGzK6ieUBT203t7RIK/fb5H+6+GyDxflaKMiebZO4B4D+Bg13ZyDTrbJ8BMLNM4ErgD13Uzs7o8kkFT0Gd6XOya4C/ufsnXdTOdOpwn82sP3AXsKgb2tmlokzDEBQz+z1wdopd90StIsU2N7MC4Hx3/2Hr84Q9rav6nFR/L+AZYLm772h/C7tcl08qeArqTJ+bd5rl0vzdGVPT2K6u1Jk+LwIecfcDiYH/aUuh34q7TznRPjN7z8zOcffdZnYO8H6KYg3ApKT14cCfgC8DF5vZTpp/7meZ2Z/cfRI9rAv7fNRKoM7dl6WhuV2hWycVPEV0ps+Y2XDgOeBGd9/e9c1Ni870+VJgtpktATKBI2b2L3f/X13f7DTr6YsKp9MLeIhjL2ouSVFmMPA2zRcysxLLg1uVGcnpcyG3U32m+frFWuBzPd2Xk/SxF83nakfx7wt8ua3K3M6xF/jWJJZzOfZC7g5Ojwu5nelzZqL8NT3dj+7qc6sy93MaX8jt8QacTi+az2f+AahLvB8Nthjwf5LKfYvmC3r1wM0p6jmdQr/DfaZ5JOXAW8DmxOvbPd2nE/Tz68B/0Xx3xz2JbWXAjMRyP5rv2qineabY0UnH3pM4bhun4N1J6e4zcC/wcdLvdDNwVk/3p6t/z0l1nNahr2kYREQCort3REQCotAXEQmIQl9EJCAKfRGRgCj0RUQCotAXEQmIQl9EJCD/DeRalSSdHz1HAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss.detach().cpu().numpy(), label = \"Training loss\")\n",
    "plt.plot(valid_loss.detach().cpu().numpy(), label = \"Validation loss\")\n",
    "plt.legend(frameon = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define on TestDataset\n",
    "class TestDataset(torch.utils.data.Dataset):\n",
    "    \n",
    "    def __init__(self, test_tokens, test_pad_masks):\n",
    "        \n",
    "        super(TestDataset, self).__init__()\n",
    "        self.test_tokens = test_tokens\n",
    "        self.test_pad_masks = test_pad_masks\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        \n",
    "        tokens = self.test_tokens[index]\n",
    "        masks = self.test_pad_masks[index]\n",
    "        \n",
    "        return (tokens, masks)\n",
    "    \n",
    "    def __len__(self,):\n",
    "        \n",
    "        return len(self.test_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tokens = []\n",
    "test_pad_masks = []\n",
    "for text in test.text:\n",
    "    tokens, masks = bert_encode(text)\n",
    "    test_tokens.append(tokens)\n",
    "    test_pad_masks.append(masks)\n",
    "    \n",
    "test_tokens = np.array(test_tokens)\n",
    "test_pad_masks = np.array(test_pad_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = TestDataset(\n",
    "    test_tokens=test_tokens,\n",
    "    test_pad_masks=test_pad_masks\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataloader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=3, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 20.00 MiB (GPU 0; 4.00 GiB total capacity; 3.04 GiB already allocated; 10.47 MiB free; 5.80 MiB cached)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-50-a17a936a9e8b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m     y_pred = model(\n\u001b[0;32m      6\u001b[0m                 \u001b[0mtokens\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m                 \u001b[0mmasks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlong\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      8\u001b[0m             )\n\u001b[0;32m      9\u001b[0m     \u001b[0my_preds\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-6-8c606aa7294c>\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, ids, masks)\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mids\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmasks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbase_model\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mids\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmasks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfc1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\transformers\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask)\u001b[0m\n\u001b[0;32m    738\u001b[0m                                        \u001b[0mhead_mask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mhead_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    739\u001b[0m                                        \u001b[0mencoder_hidden_states\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mencoder_hidden_states\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 740\u001b[1;33m                                        encoder_attention_mask=encoder_extended_attention_mask)\n\u001b[0m\u001b[0;32m    741\u001b[0m         \u001b[0msequence_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mencoder_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    742\u001b[0m         \u001b[0mpooled_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpooler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msequence_output\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\transformers\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask)\u001b[0m\n\u001b[0;32m    384\u001b[0m                 \u001b[0mall_hidden_states\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mall_hidden_states\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    385\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 386\u001b[1;33m             \u001b[0mlayer_outputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer_module\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhead_mask\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoder_hidden_states\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoder_attention_mask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    387\u001b[0m             \u001b[0mhidden_states\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    388\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\transformers\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask)\u001b[0m\n\u001b[0;32m    364\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moutputs\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mcross_attention_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m  \u001b[1;31m# add cross attentions if we output attention weights\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    365\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 366\u001b[1;33m         \u001b[0mintermediate_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mintermediate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mattention_output\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    367\u001b[0m         \u001b[0mlayer_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mintermediate_output\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattention_output\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    368\u001b[0m         \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mlayer_output\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 541\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    542\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\transformers\\modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, hidden_states)\u001b[0m\n\u001b[0;32m    326\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhidden_states\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    327\u001b[0m         \u001b[0mhidden_states\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdense\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 328\u001b[1;33m         \u001b[0mhidden_states\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mintermediate_act_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    329\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mhidden_states\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    330\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\transformers\\modeling_bert.py\u001b[0m in \u001b[0;36mgelu\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m    131\u001b[0m         \u001b[0mAlso\u001b[0m \u001b[0msee\u001b[0m \u001b[0mhttps\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m//\u001b[0m\u001b[0marxiv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0morg\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mabs\u001b[0m\u001b[1;33m/\u001b[0m\u001b[1;36m1606.08415\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    132\u001b[0m     \"\"\"\n\u001b[1;32m--> 133\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mx\u001b[0m \u001b[1;33m*\u001b[0m \u001b[1;36m0.5\u001b[0m \u001b[1;33m*\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;36m1.0\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2.0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    134\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    135\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 20.00 MiB (GPU 0; 4.00 GiB total capacity; 3.04 GiB already allocated; 10.47 MiB free; 5.80 MiB cached)"
     ]
    }
   ],
   "source": [
    "model.eval() # Turn-off drop-out to prevent overfitting\n",
    "y_preds = []\n",
    "for (tokens, masks) in test_dataloader:\n",
    "\n",
    "    y_pred = model(\n",
    "                tokens.long().to(device), \n",
    "                masks.long().to(device), \n",
    "            )\n",
    "    y_preds += y_pred.detach().cpu().numpy().squeeze().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_df = pd.read_csv(os.path.join(path_to_dataset, 'sample_submission.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_df['target'] = (np.array(y_preds) > 0).astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1971\n",
       "1    1292\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_df.target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target\n",
       "0   0       1\n",
       "1   2       1\n",
       "2   3       1\n",
       "3   9       1\n",
       "4  11       1"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_df.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}