lstm_clf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "from torchtext.vocab import build_vocab_from_iterator\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torch.utils.data import Dataset, DataLoader\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Input:\n",
    "- Context : paragraph that contains answer\n",
    "- Question : the query answerable question\n",
    "\n",
    "Output:\n",
    "- Answer: answer span"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "qa_dataset = [\n",
    "    {\n",
    "        'context': 'My name is Thang and I am from Vietnam',\n",
    "        'question': 'What is my name?',\n",
    "        'answer': 'Thang'\n",
    "    },\n",
    "    {\n",
    "        'context': 'I love painting and my favorite artist is Vincent Van Gough',\n",
    "        'question': 'What is my favorite activity?',\n",
    "        'answer' : 'painting'\n",
    "    },\n",
    "    {\n",
    "        'context' : 'I am studying Computer Science at Nanyang Technological University',\n",
    "        'question' : 'What am I studying?',\n",
    "        'answer' : 'Computer Science'\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i', 'love', 'cats']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#define tokenizer\n",
    "tokenizer = get_tokenizer('basic_english')\n",
    "\n",
    "text = 'I love cats'\n",
    "tokenizer(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'<sep>': 4,\n",
       " 'technological': 25,\n",
       " '<bos>': 2,\n",
       " '<unk>': 0,\n",
       " 'artist': 16,\n",
       " '<eos>': 3,\n",
       " '<pad>': 1,\n",
       " 'i': 5,\n",
       " 'is': 6,\n",
       " 'am': 9,\n",
       " 'my': 7,\n",
       " '?': 8,\n",
       " 'what': 10,\n",
       " 'name': 13,\n",
       " 'and': 11,\n",
       " 'favorite': 12,\n",
       " 'studying': 14,\n",
       " 'activity': 15,\n",
       " 'at': 17,\n",
       " 'computer': 18,\n",
       " 'from': 19,\n",
       " 'gough': 20,\n",
       " 'love': 21,\n",
       " 'nanyang': 22,\n",
       " 'painting': 23,\n",
       " 'science': 24,\n",
       " 'thang': 26,\n",
       " 'university': 27,\n",
       " 'van': 28,\n",
       " 'vietnam': 29,\n",
       " 'vincent': 30}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def yield_tokens(data):\n",
    "    for item in data:\n",
    "        yield(tokenizer(item['context'] + ' ' + item['question']))\n",
    "\n",
    "#create vocab\n",
    "vocab = build_vocab_from_iterator(\n",
    "    yield_tokens(qa_dataset),\n",
    "    specials=['<unk>', '<pad>', '<bos>', '<eos>', '<sep>']\n",
    ")\n",
    "vocab.set_default_index(vocab['<unk>'])\n",
    "vocab.get_stoi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 'Thang', 1: 'painting', 2: 'Computer Science'}\n",
      "{'Thang': 0, 'painting': 1, 'Computer Science': 2}\n"
     ]
    }
   ],
   "source": [
    "classes = set([item['answer'] for item in qa_dataset])\n",
    "classes_to_idx = {\n",
    "    cls_name: idx for idx, cls_name in enumerate(classes)\n",
    "}\n",
    "idx_to_classes = {\n",
    "    idx: cls_name for idx, cls_name in enumerate(classes)\n",
    "}\n",
    "print(idx_to_classes)\n",
    "print(classes_to_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "PAD_IDX = 1\n",
    "def pad_and_truncate(input_ids, max_seq_len):\n",
    "    if len(input_ids) > max_seq_len:\n",
    "        input_ids = input_ids[:max_seq_len]\n",
    "    elif len(input_ids) < max_seq_len:\n",
    "        input_ids += [PAD_IDX] * (max_seq_len - len(input_ids))\n",
    "\n",
    "    return input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize(question, context, max_seq_len):\n",
    "    input_question_ids = [vocab[token] for token in tokenizer(question)]\n",
    "    input_context_ids = [vocab[token] for token in tokenizer(context)]\n",
    "\n",
    "    input_question_ids = pad_and_truncate(input_question_ids, max_seq_len)\n",
    "    input_context_ids = pad_and_truncate(input_context_ids, max_seq_len)\n",
    "\n",
    "    input_question_ids = torch.tensor(input_question_ids, dtype=torch.long)\n",
    "    input_context_ids = torch.tensor(input_context_ids, dtype=torch.long)\n",
    "\n",
    "    return input_question_ids, input_context_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 7, 13,  6, 26, 11,  5,  9, 19, 29,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
      "         1,  1])\n",
      "tensor([10,  6,  7, 13,  8,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
      "         1,  1])\n"
     ]
    }
   ],
   "source": [
    "input_question_ids, input_context_ids = vectorize(\n",
    "    qa_dataset[0]['question'],\n",
    "    qa_dataset[0]['context'],\n",
    "    20\n",
    ")\n",
    "print(input_context_ids)\n",
    "print(input_question_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QADataset(nn.Module):\n",
    "    def __init__(self, data):\n",
    "        self.data = data\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        item = self.data[idx]\n",
    "        question_text = item['question']\n",
    "        context_text = item['context']\n",
    "\n",
    "        input_question_ids, input_context_ids = vectorize(\n",
    "            question_text, context_text, max_seq_len=20\n",
    "        )\n",
    "\n",
    "        answer_text = item['answer']\n",
    "        answer_id = classes_to_idx[answer_text]\n",
    "        answer_id = torch.tensor(answer_id, dtype=torch.long)\n",
    "\n",
    "        return input_question_ids, input_context_ids, answer_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode(input_ids):\n",
    "    return ' '.join([vocab.lookup_token(token) for token in input_ids])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[10,  6,  7, 12, 15,  8,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
      "          1,  1],\n",
      "        [10,  6,  7, 13,  8,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
      "          1,  1]]) tensor([[ 5, 21, 23, 11,  7, 12, 16,  6, 30, 28, 20,  1,  1,  1,  1,  1,  1,  1,\n",
      "          1,  1],\n",
      "        [ 7, 13,  6, 26, 11,  5,  9, 19, 29,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
      "          1,  1]]) tensor([1, 0])\n",
      "tensor([[10,  9,  5, 14,  8,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
      "          1,  1]]) tensor([[ 5,  9, 14, 18, 24, 17, 22, 25, 27,  1,  1,  1,  1,  1,  1,  1,  1,  1,\n",
      "          1,  1]]) tensor([2])\n"
     ]
    }
   ],
   "source": [
    "train_dataset = QADataset(qa_dataset)\n",
    "train_loader = DataLoader(train_dataset, batch_size=2, shuffle=True)\n",
    "\n",
    "for batch in train_loader:\n",
    "    q, c, a = batch\n",
    "    print(q, c, a)\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model: bidirectional lstm\n",
    "class QAModel(nn.Module):\n",
    "    def __init__(self, \n",
    "                 vocab_size, embedding_dim, hidden_size,\n",
    "                 n_layers, n_classes):\n",
    "        super().__init__()\n",
    "        self.questsion_embedding = nn.Embedding(\n",
    "            vocab_size, embedding_dim\n",
    "        )\n",
    "        self.context_embedding = nn.Embedding(\n",
    "            vocab_size, embedding_dim\n",
    "        )\n",
    "\n",
    "        self.lstm = nn.LSTM(\n",
    "            embedding_dim*2, hidden_size,\n",
    "            num_layers=n_layers,\n",
    "            batch_first=True,\n",
    "            bidirectional=True\n",
    "        )\n",
    "        self.fc = nn.Linear(hidden_size * 2, n_classes)\n",
    "\n",
    "    def forward(self, question, context):\n",
    "        question_embed = self.questsion_embedding(question)\n",
    "        context_embed = self.context_embedding(context)\n",
    "\n",
    "        question_n_context = torch.cat(\n",
    "            (question_embed, context_embed),\n",
    "            dim=1\n",
    "        )\n",
    "\n",
    "        lstm_out, _ = self.lstm(question_n_context)\n",
    "        lstm_out = lstm_out[:, -1, :]\n",
    "\n",
    "        out = self.fc(lstm_out)\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 3])\n"
     ]
    }
   ],
   "source": [
    "#Model parameters\n",
    "EMBEDDING_DIM = 32\n",
    "HIDDEN_SIZE = 128\n",
    "VOCAB_SIZE = len(vocab)\n",
    "N_LAYERS = 2\n",
    "N_CLASSES = len(classes)\n",
    "MAX_CONTEXT_LEN = 20 # max_seq_len = 20\n",
    "model = QAModel(\n",
    "    VOCAB_SIZE, EMBEDDING_DIM, HIDDEN_SIZE, N_LAYERS, N_CLASSES\n",
    ")\n",
    "\n",
    "input_context = torch.randint(0, 10, size=(1, MAX_CONTEXT_LEN))\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    logits = model(input_context, input_context)\n",
    "\n",
    "print(logits.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.1375741958618164\n",
      "1.0970170497894287\n",
      "1.1093568801879883\n",
      "1.0720338821411133\n",
      "1.037621021270752\n",
      "1.2168563604354858\n",
      "1.1056877374649048\n",
      "1.0685009956359863\n",
      "1.0238081216812134\n",
      "1.171850323677063\n",
      "1.0436911582946777\n",
      "1.0676417350769043\n",
      "1.075181007385254\n",
      "0.8446162939071655\n",
      "0.8897265195846558\n",
      "1.024154543876648\n",
      "0.9852239489555359\n",
      "0.4325188994407654\n",
      "0.9271750450134277\n",
      "0.2334124594926834\n",
      "0.491590678691864\n",
      "0.9268525242805481\n",
      "0.8360755443572998\n",
      "0.04568187892436981\n",
      "0.3978568911552429\n",
      "0.836230993270874\n",
      "0.40441057085990906\n",
      "0.8005020022392273\n",
      "0.36667779088020325\n",
      "0.8077053427696228\n",
      "0.39257487654685974\n",
      "0.7396137118339539\n",
      "0.368089884519577\n",
      "0.7426681518554688\n",
      "0.7229405641555786\n",
      "0.0021495348773896694\n",
      "0.382950097322464\n",
      "0.6704594492912292\n",
      "0.7136815786361694\n",
      "0.001401038491167128\n"
     ]
    }
   ],
   "source": [
    "LR = 1e-3\n",
    "EPOCHS = 20\n",
    "optimizer = torch.optim.Adam(\n",
    "    model.parameters(), lr=LR\n",
    ")\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "model.train()\n",
    "for _ in range(EPOCHS):\n",
    "    for idx, (input_question_ids, input_context_ids, answer_id) in enumerate(train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        ouputs = model(input_question_ids, input_context_ids)\n",
    "        loss = criterion(ouputs, answer_id)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        print(loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context: My name is Thang and I am from Vietnam\n",
      "Question: What is my name?\n",
      "Prediction: Thang\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    sample = qa_dataset[0]\n",
    "    context, question, answer = sample.values()\n",
    "    question_ids, context_ids = vectorize(question, context, MAX_CONTEXT_LEN)\n",
    "    question_ids = question_ids.unsqueeze(0)\n",
    "    context_ids = context_ids.unsqueeze(0)\n",
    "    outputs = model(question_ids, context_ids)\n",
    "    _, predicted = torch.max(outputs.data, 1)\n",
    "    print(f'Context: {context}')\n",
    "    print(f'Question: {question}')\n",
    "    print(f'Prediction: {idx_to_classes[predicted.numpy()[0]]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "devinthang",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}