train_pretrained_tox_1cat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Apr 24 02:17:59 2020       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 435.21       Driver Version: 435.21       CUDA Version: 10.1     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  GeForce RTX 208...  Off  | 00000000:1A:00.0 Off |                  N/A |\n",
      "| 27%   24C    P8    13W / 260W |    318MiB / 11019MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  GeForce RTX 208...  Off  | 00000000:1B:00.0 Off |                  N/A |\n",
      "| 27%   23C    P8     9W / 260W |    165MiB / 11019MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   2  GeForce RTX 208...  Off  | 00000000:1C:00.0 Off |                  N/A |\n",
      "| 27%   26C    P8     5W / 260W |   9215MiB / 11019MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   3  GeForce RTX 208...  Off  | 00000000:1E:00.0 Off |                  N/A |\n",
      "| 27%   25C    P8     9W / 260W |   6643MiB / 11019MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   4  GeForce RTX 208...  Off  | 00000000:3D:00.0 Off |                  N/A |\n",
      "| 27%   24C    P2    46W / 260W |    165MiB / 11019MiB |     21%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   5  GeForce RTX 208...  Off  | 00000000:3E:00.0 Off |                  N/A |\n",
      "| 27%   25C    P8    14W / 260W |    165MiB / 11019MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   6  GeForce RTX 208...  Off  | 00000000:40:00.0 Off |                  N/A |\n",
      "| 27%   24C    P8     8W / 260W |    165MiB / 11019MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   7  GeForce RTX 208...  Off  | 00000000:41:00.0 Off |                  N/A |\n",
      "| 27%   24C    P8    14W / 260W |   2314MiB / 11019MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "|    0     11997      C   /opt/.pyenv/versions/3.7.4/bin/python        155MiB |\n",
      "|    0     19866      C   /opt/.pyenv/versions/3.7.4/bin/python3.7     153MiB |\n",
      "|    1     11997      C   /opt/.pyenv/versions/3.7.4/bin/python        155MiB |\n",
      "|    2      2716      C   /opt/.pyenv/versions/3.7.4/bin/python       1843MiB |\n",
      "|    2      7213      C   /opt/.pyenv/versions/3.7.4/bin/python3      1335MiB |\n",
      "|    2      7485      C   /home/voronov/anaconda3/bin/python          4537MiB |\n",
      "|    2     11997      C   /opt/.pyenv/versions/3.7.4/bin/python        155MiB |\n",
      "|    2     29221      C   python3                                     1335MiB |\n",
      "|    3      7486      C   /home/voronov/anaconda3/bin/python          5521MiB |\n",
      "|    3     11997      C   /opt/.pyenv/versions/3.7.4/bin/python        155MiB |\n",
      "|    3     27465      C   /opt/.pyenv/versions/3.7.4/bin/python        957MiB |\n",
      "|    4     11997      C   /opt/.pyenv/versions/3.7.4/bin/python        155MiB |\n",
      "|    5     11997      C   /opt/.pyenv/versions/3.7.4/bin/python        155MiB |\n",
      "|    6     11997      C   /opt/.pyenv/versions/3.7.4/bin/python        155MiB |\n",
      "|    7     11997      C   /opt/.pyenv/versions/3.7.4/bin/python        155MiB |\n",
      "|    7     32430      C   /opt/.pyenv/versions/3.7.4/bin/python       2149MiB |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from transformers import BertTokenizer, BertForSequenceClassification\n",
    "import torch\n",
    "import re\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "device = torch.device('cuda:4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_path='train.csv'\n",
    "train = pd.read_csv(train_data_path)\n",
    "test_texts = pd.read_csv('test.csv')\n",
    "test_labels = pd.read_csv('test_labels.csv')\n",
    "\n",
    "test_labels = test_labels.iloc[:,1:]\n",
    "test = test_texts.join(test_labels)\n",
    "test = test[test['toxic']!=-1]\n",
    "\n",
    "train_texts = train.iloc[:,1].values\n",
    "train_labels = train.iloc[:,2:].values\n",
    "\n",
    "test_texts = test.iloc[:,1].values\n",
    "test_labels = test.iloc[:,2:].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0000997932d777bf</td>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>000103f0d9cfb60f</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>000113f07ec002fd</td>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0001b41b1c6bb37e</td>\n",
       "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0001d958c54c6e35</td>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  toxic  \\\n",
       "0  0000997932d777bf  Explanation\\nWhy the edits made under my usern...      0   \n",
       "1  000103f0d9cfb60f  D'aww! He matches this background colour I'm s...      0   \n",
       "2  000113f07ec002fd  Hey man, I'm really not trying to edit war. It...      0   \n",
       "3  0001b41b1c6bb37e  \"\\nMore\\nI can't make any real suggestions on ...      0   \n",
       "4  0001d958c54c6e35  You, sir, are my hero. Any chance you remember...      0   \n",
       "\n",
       "   severe_toxic  obscene  threat  insult  identity_hate  \n",
       "0             0        0       0       0              0  \n",
       "1             0        0       0       0              0  \n",
       "2             0        0       0       0              0  \n",
       "3             0        0       0       0              0  \n",
       "4             0        0       0       0              0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['explanation', 'why', 'the', 'edit', '##s', 'made', 'under', 'my', 'user', '##name', 'hardcore', 'metallic', '##a', 'fan', 'were', 'reverted', '?', 'they', 'weren', \"'\", 't', 'van', '##dal', '##isms', ',', 'just', 'closure', 'on', 'some', 'gas', 'after', 'i', 'voted', 'at', 'new', 'york', 'dolls', 'fa', '##c', '.', 'and', 'please', 'don', \"'\", 't', 'remove', 'the', 'template', 'from', 'the', 'talk', 'page', 'since', 'i', \"'\", 'm', 'retired', 'now', '.', '89', '.', '205', '.', '38', '.', '27']\n"
     ]
    }
   ],
   "source": [
    "check_line = tokenizer.tokenize(train_texts[0])\n",
    "print(check_line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = train_labels[:,0]\n",
    "test_labels = test_labels[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize_elem(text, label, max_len=128):\n",
    "    \n",
    "    new_words = tokenizer.tokenize(text)\n",
    "    input_masks = [1]*len(new_words)\n",
    "\n",
    "    new_words = ['[CLS]'] + new_words[:max_len-2] + ['[SEP]']\n",
    "    input_masks = [1] + input_masks[:max_len-2] + [1]\n",
    "  \n",
    "    input_ids = tokenizer.convert_tokens_to_ids(new_words)\n",
    "    #right padding\n",
    "    padding = [0] * (max_len - len(input_ids))\n",
    "    input_ids += padding \n",
    "    input_masks += padding\n",
    "\n",
    "    assert len(input_ids) == max_len\n",
    "    assert len(input_masks) == max_len\n",
    "\n",
    "    label_ids = [label]\n",
    "\n",
    "    return torch.tensor(input_ids), torch.tensor(input_masks), torch.tensor(label_ids)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(X, y, n):\n",
    "    train, attention_masks_train, y_train = [], [], []\n",
    "    data_train=[]\n",
    "    for i, (t, l) in enumerate(zip(X, y)):\n",
    "        if i%10000 == 0:\n",
    "            print(i)\n",
    "        ids, masks, labels = vectorize_elem(t, l, max_len=n)\n",
    "        train.append(ids)\n",
    "        attention_masks_train.append(masks)\n",
    "        y_train.append(labels)\n",
    "\n",
    "    for i in range(len(train)):\n",
    "        data_train.append((train[i], attention_masks_train[i], y_train[i]))\n",
    "\n",
    "    return data_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "70000\n",
      "80000\n",
      "90000\n",
      "100000\n",
      "110000\n",
      "120000\n",
      "130000\n",
      "140000\n",
      "150000\n",
      "finished preprocessing of train set\n",
      "0\n",
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "finished preprocessing of test set\n",
      "CPU times: user 4min 9s, sys: 1.1 s, total: 4min 11s\n",
      "Wall time: 4min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "MAX_LEN = 128\n",
    "X_train = preprocess(train_texts, train_labels,n=MAX_LEN)\n",
    "print('finished preprocessing of train set')\n",
    "X_test = preprocess(test_texts, test_labels, n=MAX_LEN)\n",
    "print('finished preprocessing of test set')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(preds, labs):\n",
    "    num_right = torch.sum(preds == labs).item()\n",
    "    return num_right/preds.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def right(preds, labs):\n",
    "    #just a number of right answers in the batch\n",
    "    num_right = torch.sum(preds == labs)\n",
    "    return num_right.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_epoch(model, train_dataset, test_dataset, bs_train, bs_test):\n",
    "    train_losses = []\n",
    "    train_right_answers = 0\n",
    "    test_losses = []\n",
    "    test_accuracies = []\n",
    "    \n",
    "   \n",
    "    train_loader = DataLoader(train_dataset, batch_size=bs_train, shuffle=True)\n",
    "    test_loader = DataLoader(test_dataset, batch_size=bs_test, shuffle=False)\n",
    "\n",
    "    for j, train_batch, in enumerate(train_loader):\n",
    "        if j%1000==9:\n",
    "            print(j)\n",
    "        #train\n",
    "        model.train()\n",
    "        b_ids, b_mask, b_labels = train_batch\n",
    "        b_ids = b_ids.to(device)\n",
    "        b_mask = b_mask.to(device)\n",
    "        b_labels = b_labels.to(device)\n",
    "\n",
    "        model.zero_grad()\n",
    "\n",
    "        output = model(b_ids, token_type_ids=None, attention_mask=b_mask, labels=b_labels)\n",
    "        loss = output[0]\n",
    "        logits = output[1]\n",
    "\n",
    "        train_losses.append(loss.item())\n",
    "        train_right_answers += right(logits, b_labels)\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        lr_scheduler.step()\n",
    "\n",
    "        #test\n",
    "        #model.eval()\n",
    "#\n",
    "        #b_ids, b_mask, b_labels = test_batch\n",
    "#\n",
    "        #b_ids = b_ids.to(device)\n",
    "        #b_mask = b_mask.to(device)\n",
    "        #b_labels = b_labels.to(device)\n",
    "        if j % 500 == 9:\n",
    "            model.eval()\n",
    "            test_loss = 0\n",
    "            test_right_answer = 0\n",
    "            for test_batch in test_loader:\n",
    "                test_ids, test_mask, test_labels = test_batch\n",
    "                test_ids = test_ids.to(device)\n",
    "                test_mask = test_mask.to(device)\n",
    "                test_labels = test_labels.to(device)\n",
    "                with torch.no_grad():\n",
    "                    output = model(test_ids, token_type_ids=None, attention_mask=test_mask, labels=test_labels)\n",
    "                    loss = output[0]\n",
    "                    logits = output[1]\n",
    "\n",
    "                    test_loss += loss.item()\n",
    "                    test_right_answer += right(logits, test_labels)\n",
    "            test_losses.append(test_loss)\n",
    "            test_accuracies.append(test_right_answer/len(test_dataset))\n",
    "    \n",
    "  \n",
    "    print(\"Last average batch_loss on train: {}, \\t Accuracy: {}\".format(train_losses[-1], \n",
    "                                                                         train_right_answers/(j+1)/bs))\n",
    "    print(\"Last average batch_loss on test: {}, \\t Accuracy: {}\".format(test_losses[-1], \n",
    "                                                                        test_right_answer/len(test_dataset)))\n",
    "    return train_losses, train_right_answers/(j+1)/bs, test_losses, test_accuracies\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "1009\n",
      "2009\n",
      "3009\n",
      "4009\n",
      "5009\n",
      "6009\n",
      "7009\n",
      "8009\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from torch.utils.data import DataLoader\n",
    "from transformers import AdamW, get_linear_schedule_with_warmup\n",
    "\n",
    "#for lr in [1e-3, 5e-4, 1e-4, 5e-5, 1e-5, 5e-6, 1e-6]:\n",
    "BATCH_SIZE = 16\n",
    "BATCH_SIZE_TEST = 64\n",
    "MAX_N_EPOCHS = 3\n",
    "LEARNING_RATE = 2e-5\n",
    "\n",
    "\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=2).to(device)\n",
    "optimizer = AdamW(model.parameters(), lr=LEARNING_RATE)\n",
    "lr_scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0.1,\n",
    "                                               num_training_steps=(len(X_train) // BATCH_SIZE)*MAX_N_EPOCHS)\n",
    "#lr_scheduler = WarmupLinearSchedule(optimizer, warmup_steps=0.1,\n",
    "                                    #t_total=(len(X_train) // BATCH_SIZE)*MAX_N_EPOCHS)\n",
    "\n",
    "train_losses = []\n",
    "train_accs = []\n",
    "test_losses = []\n",
    "test_accs = []\n",
    "n_batches = len(X_train)//BATCH_SIZE\n",
    "\n",
    "for i in range(MAX_N_EPOCHS):\n",
    "    stats = train_epoch(model, X_train, X_test, bs_train=BATCH_SIZE, bs_test=BATCH_SIZE_TEST)\n",
    "    print(f\"Epoch {i} finished.\")\n",
    "    train_losses += stats[0]\n",
    "    train_accs.append(stats[1])\n",
    "    test_losses += stats[2]\n",
    "    test_accs += stats[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.figure(figsize=[20., 10.])\n",
    "plt.subplot(211)\n",
    "plt.title(\"Average batch loss\")\n",
    "plt.plot(train_losses, c='b', linewidth=0.7, label='train')\n",
    "plt.plot(test_losses, c='g', linewidth=0.7, label='dev')\n",
    "for x in [len(X_test)//BATCH_SIZE_TEST*(i+1) for i in range(MAX_N_EPOCHS-1)]:\n",
    "    plt.axvline(x, c='black', linewidth=2)\n",
    "plt.xlabel(\"batch number\")\n",
    "plt.ylabel(\"loss\")\n",
    "plt.legend()\n",
    "\n",
    "plt.subplot(212)\n",
    "plt.title(\"Accuracy on epoch\")\n",
    "plt.plot(train_accs, c='b', label='train')\n",
    "plt.plot(dev_accs, c='g', label='dev')\n",
    "plt.xlabel(\"epoch number\")\n",
    "plt.ylabel(\"accuracy\")\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = DataLoader(X_train, batch_size=4, shuffle=True)\n",
    "for i, train_batch in enumerate(p):\n",
    "    a, b, c = train_batch\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 128])\n"
     ]
    }
   ],
   "source": [
    "print(a.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 128])\n"
     ]
    }
   ],
   "source": [
    "print(b.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 4, 6])\n"
     ]
    }
   ],
   "source": [
    "print(c.unsqueeze(0).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 6])\n"
     ]
    }
   ],
   "source": [
    "print(c.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_test = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Expected input batch_size (4) to match target batch_size (24).",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-51-4ae9f69a5a1d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/transformers/modeling_bert.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, labels)\u001b[0m\n\u001b[1;32m   1207\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1208\u001b[0m                 \u001b[0mloss_fct\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCrossEntropyLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1209\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_fct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_labels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1210\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    530\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/loss.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, target)\u001b[0m\n\u001b[1;32m    914\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    915\u001b[0m         return F.cross_entropy(input, target, weight=self.weight,\n\u001b[0;32m--> 916\u001b[0;31m                                ignore_index=self.ignore_index, reduction=self.reduction)\n\u001b[0m\u001b[1;32m    917\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    918\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mcross_entropy\u001b[0;34m(input, target, weight, size_average, ignore_index, reduce, reduction)\u001b[0m\n\u001b[1;32m   2019\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0msize_average\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mreduce\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2020\u001b[0m         \u001b[0mreduction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_Reduction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_get_string\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize_average\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2021\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mnll_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_softmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2022\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2023\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mnll_loss\u001b[0;34m(input, target, weight, size_average, ignore_index, reduce, reduction)\u001b[0m\n\u001b[1;32m   1834\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1835\u001b[0m         raise ValueError('Expected input batch_size ({}) to match target batch_size ({}).'\n\u001b[0;32m-> 1836\u001b[0;31m                          .format(input.size(0), target.size(0)))\n\u001b[0m\u001b[1;32m   1837\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mdim\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1838\u001b[0m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnll_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_Reduction\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_enum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Expected input batch_size (4) to match target batch_size (24)."
     ]
    }
   ],
   "source": [
    "out = model_test(a, token_type_ids=None, attention_mask=b, labels=)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}