BERT_to_the_rescue.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT to the rescue.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_FXneEfpdMM",
        "colab_type": "code",
        "outputId": "ff8057f1-23dc-49b8-b11c-f68823bf6f90",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "!pip install pytorch_pretrained_bert pytorch-nlp"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pytorch_pretrained_bert in /usr/local/lib/python3.6/dist-packages (0.6.2)\n",
            "Requirement already satisfied: pytorch-nlp in /usr/local/lib/python3.6/dist-packages (0.4.1)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (2018.1.10)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.16.4)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.9.157)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (2.21.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (4.28.1)\n",
            "Requirement already satisfied: torch>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.1.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from pytorch-nlp) (0.24.2)\n",
            "Requirement already satisfied: botocore<1.13.0,>=1.12.157 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (1.12.157)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (0.2.0)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (0.9.4)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (2.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (2019.3.9)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (1.24.3)\n",
            "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python3.6/dist-packages (from pandas->pytorch-nlp) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python3.6/dist-packages (from pandas->pytorch-nlp) (2.5.3)\n",
            "Requirement already satisfied: docutils>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.157->boto3->pytorch_pretrained_bert) (0.14)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.5.0->pandas->pytorch-nlp) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "3c8e3d88-13b0-4652-973d-3fa2c023e48e",
        "id": "EnVIV6Vt8f4d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import sys\n",
        "import numpy as np\n",
        "import random as rn\n",
        "import torch\n",
        "from pytorch_pretrained_bert import BertModel\n",
        "from torch import nn\n",
        "from torchnlp.datasets import imdb_dataset\n",
        "from pytorch_pretrained_bert import BertTokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from torch.optim import Adam\n",
        "from torch.nn.utils import clip_grad_norm_\n",
        "from IPython.display import clear_output"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cUYrv06z8gaF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "rn.seed(321)\n",
        "np.random.seed(321)\n",
        "torch.manual_seed(321)\n",
        "torch.cuda.manual_seed(321)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rgkbhHcB17GY",
        "colab_type": "text"
      },
      "source": [
        "## Prepare the Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ur8i7boP6qtb",
        "colab": {}
      },
      "source": [
        "train_data, test_data = imdb_dataset(train=True, test=True)\n",
        "rn.shuffle(train_data)\n",
        "rn.shuffle(test_data)\n",
        "train_data = train_data[:1000]\n",
        "test_data = test_data[:100]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l1VENo6tqG7J",
        "colab_type": "code",
        "outputId": "5f10799f-d17d-42fd-a4c2-c760b61f7ed8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_texts, train_labels = list(zip(*map(lambda d: (d['text'], d['sentiment']), train_data)))\n",
        "test_texts, test_labels = list(zip(*map(lambda d: (d['text'], d['sentiment']), test_data)))\n",
        "\n",
        "len(train_texts), len(train_labels), len(test_texts), len(test_labels)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 1000, 100, 100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ty24UrRjqIsb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "26trq3gIrJeG",
        "colab_type": "code",
        "outputId": "196b2aa3-0176-4010-baf8-e650ca0da658",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tokenizer.tokenize('Hi my name is Dima')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['hi', 'my', 'name', 'is', 'dim', '##a']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1k9rcOzQr5Zm",
        "colab_type": "code",
        "outputId": "b9b59a4e-57bd-4e9c-fcd1-2a4ba1fadc0e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_tokens = list(map(lambda t: ['[CLS]'] + tokenizer.tokenize(t)[:510] + ['[SEP]'], train_texts))\n",
        "test_tokens = list(map(lambda t: ['[CLS]'] + tokenizer.tokenize(t)[:510] + ['[SEP]'], test_texts))\n",
        "\n",
        "len(train_tokens), len(test_tokens)                   \n",
        "                   "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Ca7KKnhuT5c",
        "colab_type": "code",
        "outputId": "69ba3b87-4d86-448b-c9cb-7be8361182f6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_tokens_ids = pad_sequences(list(map(tokenizer.convert_tokens_to_ids, train_tokens)), maxlen=512, truncating=\"post\", padding=\"post\", dtype=\"int\")\n",
        "test_tokens_ids = pad_sequences(list(map(tokenizer.convert_tokens_to_ids, test_tokens)), maxlen=512, truncating=\"post\", padding=\"post\", dtype=\"int\")\n",
        "\n",
        "train_tokens_ids.shape, test_tokens_ids.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((1000, 512), (100, 512))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "a5447340-6da3-4f2c-d284-098c4c906c47",
        "id": "F7POtHuIOV-6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_y = np.array(train_labels) == 'pos'\n",
        "test_y = np.array(test_labels) == 'pos'\n",
        "train_y.shape, test_y.shape, np.mean(train_y), np.mean(test_y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((1000,), (100,), 0.489, 0.5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U-xXMEqXOWTE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_masks = [[float(i > 0) for i in ii] for ii in train_tokens_ids]\n",
        "test_masks = [[float(i > 0) for i in ii] for ii in test_tokens_ids]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2K4JQMFo1-_S",
        "colab_type": "text"
      },
      "source": [
        "# Baseline"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wdzjl_WlwpKr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.metrics import classification_report"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9jyb-hJ0xAgG",
        "colab_type": "code",
        "outputId": "4cc5ca8c-646f-410f-d5c5-998fdf82f00e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "baseline_model = make_pipeline(CountVectorizer(ngram_range=(1,3)), LogisticRegression()).fit(train_texts, train_labels)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q9IzjAX_2VLf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "baseline_predicted = baseline_model.predict(test_texts)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QnsCRIaQ3GPQ",
        "colab_type": "code",
        "outputId": "46417074-2870-4a52-9a6a-307bde86b704",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "source": [
        "print(classification_report(test_labels, baseline_predicted))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "         neg       0.81      0.78      0.80        50\n",
            "         pos       0.79      0.82      0.80        50\n",
            "\n",
            "    accuracy                           0.80       100\n",
            "   macro avg       0.80      0.80      0.80       100\n",
            "weighted avg       0.80      0.80      0.80       100\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r_hEhebQ3YqI",
        "colab_type": "text"
      },
      "source": [
        "# Bert Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E234ByBa3Qtb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BertBinaryClassifier(nn.Module):\n",
        "    def __init__(self, dropout=0.1):\n",
        "        super(BertBinaryClassifier, self).__init__()\n",
        "\n",
        "        self.bert = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        self.linear = nn.Linear(768, 1)\n",
        "        self.sigmoid = nn.Sigmoid()\n",
        "    \n",
        "    def forward(self, tokens, masks=None):\n",
        "        _, pooled_output = self.bert(tokens, attention_mask=masks, output_all_encoded_layers=False)\n",
        "        dropout_output = self.dropout(pooled_output)\n",
        "        linear_output = self.linear(dropout_output)\n",
        "        proba = self.sigmoid(linear_output)\n",
        "        return proba\n",
        "        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ED9SE1Ka8W9x",
        "colab_type": "code",
        "outputId": "ab8b549b-137b-491a-cf6f-f20718dd73ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "device"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FNZ_3auqDbjl",
        "colab_type": "code",
        "outputId": "1c20ee58-abb4-4bf2-f79a-851ab54c31eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'0.0M'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Sf9n8zouENRi",
        "colab": {}
      },
      "source": [
        "bert_clf = BertBinaryClassifier()\n",
        "bert_clf = bert_clf.cuda()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LbHkaJuZEkYr",
        "colab_type": "code",
        "outputId": "dff4b8e7-db91-46de-8b8c-bf0cddccaba9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'439.065088M'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LOQ-870M7VWy",
        "colab_type": "code",
        "outputId": "f7c010bf-eb70-413f-96db-c0eb14769cfa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "x = torch.tensor(train_tokens_ids[:3]).to(device)\n",
        "y, pooled = bert_clf.bert(x, output_all_encoded_layers=False)\n",
        "x.shape, y.shape, pooled.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([3, 512]), torch.Size([3, 512, 768]), torch.Size([3, 768]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LCb_pK4X7hb9",
        "colab_type": "code",
        "outputId": "77237549-1b51-4d51-a11e-84e53fa82cb3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "y = bert_clf(x)\n",
        "y.cpu().detach().numpy()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.5208096 ],\n",
              "       [0.4679125 ],\n",
              "       [0.55676025]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MUm-gFCuFkoI",
        "colab_type": "code",
        "outputId": "4cac9ca3-1080-4307-81a4-d2a7d7618eab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'6698.214912M'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KzzsUZOUFcxp",
        "colab_type": "code",
        "outputId": "fdb38786-4df8-473d-cf45-f635741a868e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "y, x, pooled = None, None, None\n",
        "torch.cuda.empty_cache()\n",
        "str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'439.930368M'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c9LPIYcn99r8",
        "colab_type": "text"
      },
      "source": [
        "# Fine-tune BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZUkXhM1k_TAl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 4\n",
        "EPOCHS = 10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jGwV0yqg_o2u",
        "colab_type": "code",
        "outputId": "236eae60-f405-4f2b-89be-1164fb924b91",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_tokens_tensor = torch.tensor(train_tokens_ids)\n",
        "train_y_tensor = torch.tensor(train_y.reshape(-1, 1)).float()\n",
        "\n",
        "test_tokens_tensor = torch.tensor(test_tokens_ids)\n",
        "test_y_tensor = torch.tensor(test_y.reshape(-1, 1)).float()\n",
        "\n",
        "train_masks_tensor = torch.tensor(train_masks)\n",
        "test_masks_tensor = torch.tensor(test_masks)\n",
        "\n",
        "str(torch.cuda.memory_allocated(device)/1000000 ) + 'M'"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'439.930368M'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Yl2JpCe9YAu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_dataset = TensorDataset(train_tokens_tensor, train_masks_tensor, train_y_tensor)\n",
        "train_sampler = RandomSampler(train_dataset)\n",
        "train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=BATCH_SIZE)\n",
        "\n",
        "test_dataset = TensorDataset(test_tokens_tensor, test_masks_tensor, test_y_tensor)\n",
        "test_sampler = SequentialSampler(test_dataset)\n",
        "test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=BATCH_SIZE)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JF_QD0naS8EQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "param_optimizer = list(bert_clf.sigmoid.named_parameters()) \n",
        "optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b28PcoDh_cyd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = Adam(bert_clf.parameters(), lr=3e-6)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z6yIChYvBP5F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        " torch.cuda.empty_cache()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqh8tCl4AFjo",
        "colab_type": "code",
        "outputId": "15c746cf-ba17-483a-d0ff-0bcbf9e96431",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "for epoch_num in range(EPOCHS):\n",
        "    bert_clf.train()\n",
        "    train_loss = 0\n",
        "    for step_num, batch_data in enumerate(train_dataloader):\n",
        "        token_ids, masks, labels = tuple(t.to(device) for t in batch_data)\n",
        "        print(str(torch.cuda.memory_allocated(device)/1000000 ) + 'M')\n",
        "        logits = bert_clf(token_ids, masks)\n",
        "        \n",
        "        loss_func = nn.BCELoss()\n",
        "\n",
        "        batch_loss = loss_func(logits, labels)\n",
        "        train_loss += batch_loss.item()\n",
        "        \n",
        "        \n",
        "        bert_clf.zero_grad()\n",
        "        batch_loss.backward()\n",
        "        \n",
        "\n",
        "        clip_grad_norm_(parameters=bert_clf.parameters(), max_norm=1.0)\n",
        "        optimizer.step()\n",
        "        \n",
        "        clear_output(wait=True)\n",
        "        print('Epoch: ', epoch_num + 1)\n",
        "        print(\"\\r\" + \"{0}/{1} loss: {2} \".format(step_num, len(train_data) / BATCH_SIZE, train_loss / (step_num + 1)))\n",
        "        "
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch:  10\n",
            "\r249/250.0 loss: 0.4731319788321853 \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GHFWhkRYHv5l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bert_clf.eval()\n",
        "bert_predicted = []\n",
        "all_logits = []\n",
        "with torch.no_grad():\n",
        "    for step_num, batch_data in enumerate(test_dataloader):\n",
        "\n",
        "        token_ids, masks, labels = tuple(t.to(device) for t in batch_data)\n",
        "\n",
        "        logits = bert_clf(token_ids, masks)\n",
        "        loss_func = nn.BCELoss()\n",
        "        loss = loss_func(logits, labels)\n",
        "        numpy_logits = logits.cpu().detach().numpy()\n",
        "        \n",
        "        bert_predicted += list(numpy_logits[:, 0] > 0.5)\n",
        "        all_logits += list(numpy_logits[:, 0])\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vg_sX9BjooL-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2e85f485-c125-4ed6-bb8a-07ed708c54f9"
      },
      "source": [
        "np.mean(bert_predicted)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.56"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-DmIJqUnkVM8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "de735d0b-ad04-4d81-ef14-c6394bab92d1"
      },
      "source": [
        "print(classification_report(test_y, bert_predicted))"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "       False       0.86      0.76      0.81        50\n",
            "        True       0.79      0.88      0.83        50\n",
            "\n",
            "    accuracy                           0.82       100\n",
            "   macro avg       0.82      0.82      0.82       100\n",
            "weighted avg       0.82      0.82      0.82       100\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eIBvoExLpOne",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}