linear_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import  load_data\n",
    "from linear_format import encode_data, extract_labels, NeuralNetwork, get_batches\n",
    "import params\n",
    "import random, torch, pickle\n",
    "from torch import nn\n",
    "import numpy as np\n",
    "from transformers import BertForSequenceClassification\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data: data/stac_data/test_data.json\n",
      "109 dialogs, 1156 edus, 1126 relations, 8 backward relations\n",
      "77 edus have multiple parents\n",
      "models/stac/bert_finetuned.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded finetuned Bert model\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(params.device)\n",
    "\n",
    "test_data = load_data(params.data_path + \"test_data.json\", map_relations=params.map_relations)\n",
    "\n",
    "input_text, labels, positions = encode_data(test_data, test = True)\n",
    "input_ids, attention_masks, token_type_ids, tokens, position_ids = extract_labels(input_text, labels, positions, token = True)\n",
    "\n",
    "position_ids = [position_id.clone().detach().to(device) for position_id in position_ids]\n",
    "\n",
    "model_path = params.model_path + params.bert_name + '.pth'\n",
    "print(model_path)\n",
    "\n",
    "embedder = BertForSequenceClassification.from_pretrained(\n",
    "    params.model_name, \n",
    "    output_attentions = False,\n",
    "    output_hidden_states = True, attention_probs_dropout_prob=0, hidden_dropout_prob=0\n",
    ")\n",
    "\n",
    "checkpoint = torch.load(model_path, map_location=params.device)\n",
    "embedder.load_state_dict(checkpoint['model_state_dict'])\n",
    "embedder.to(device)\n",
    "print('Loaded finetuned Bert model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_data = []\n",
    "for i in range(len(labels)):\n",
    "    meta_data.append([[lb[2], lb[2]-lb[1]] for lb in labels[i]]) \n",
    "\n",
    "model_path = params.model_path + params.linear_name + '.pth'\n",
    "linear = NeuralNetwork().to(device)\n",
    "checkpoint = torch.load(model_path, map_location=params.device)\n",
    "linear.load_state_dict(checkpoint['model_state_dict'])\n",
    "linear.to(device)\n",
    "linear.eval()\n",
    "\n",
    "# consider only 10 preceding edus\n",
    "indices = [max(len(input_id)-10, 0) for input_id in input_ids]\n",
    "input_ids = [input_id[-10:] for input_id in input_ids]\n",
    "attention_masks = [attention_mask[-10:] for attention_mask in attention_masks]\n",
    "token_type_ids = [token_type_id[-10:] for token_type_id in token_type_ids]\n",
    "position_ids = [position_id[-10:] for position_id in position_ids]\n",
    "\n",
    "batches = get_batches(len(input_ids), params.batch_size_linear)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 66/66 [00:16<00:00,  4.00it/s]\n"
     ]
    }
   ],
   "source": [
    "predictions = []\n",
    "targets = []\n",
    "pred_dict = {}\n",
    "for batch in tqdm(batches) : \n",
    "\n",
    "    for i in batch:\n",
    "        output = embedder(input_ids[i].to(device), \n",
    "                             token_type_ids=token_type_ids[i].to(device), \n",
    "                             attention_mask=attention_masks[i].to(device),\n",
    "                             position_ids = position_ids[i].to(device),\n",
    "                             return_dict=True)\n",
    "        H_embed = torch.stack([torch.cat((output.hidden_states[-1][cand][0],torch.tensor(meta_data[i][cand]).to(device)),0) for cand in range(len(output.hidden_states[0]))])           \n",
    "\n",
    "        H_embed = H_embed.to(device)\n",
    "        logits = linear(H_embed).unsqueeze(0) \n",
    "        m = nn.Sigmoid()\n",
    "\n",
    "        mod =(m(logits)).squeeze(-1).cpu().tolist()[0]\n",
    "        xs = [i for i in range(len(mod)) if mod[i] > 0.81]  # 0.95\n",
    "        if len(xs) == 0 : xs += [ np.argmax([float(p[0]) for p in logits[0].cpu()])]\n",
    "\n",
    "        for pred_x in xs :\n",
    "            pred_y = labels[i][pred_x][2]\n",
    "            dialog_id = labels[i][0][0]\n",
    "\n",
    "            if str(dialog_id) not in pred_dict.keys() : \n",
    "                pred_dict[str(dialog_id)] = [[pred_x + indices[i], pred_y]]\n",
    "            else :\n",
    "                pred_dict[str(dialog_id)] += [[pred_x + indices[i], pred_y]]\n",
    "\n",
    "        predictions = [pred[1] for pred in pred_dict.items()]\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nb of total predictions :  1200\n",
      "nb of relations :  1239\n",
      "nb of correct predictions :  891\n",
      "Precition : 0.7425    Recall :  0.7191283292978208\n",
      "F1 score :  0.7306273062730627\n"
     ]
    }
   ],
   "source": [
    "test_pred = predictions\n",
    "# compute the f1 score\n",
    "test_truth = []\n",
    "for dialogue in test_data:\n",
    "    truth = []\n",
    "    if len(dialogue['edus'])==1:\n",
    "        continue\n",
    "    for edu in dialogue['relations']:\n",
    "        tup = (edu['x'],edu['y'], edu['type'])\n",
    "        truth += [tup]\n",
    "    test_truth += [truth]\n",
    "    \n",
    "cnt_pred = []\n",
    "for dialog in test_pred:\n",
    "    cnt_pred += [len(dialog)+1]\n",
    "\n",
    "cnt_pred = sum(cnt_pred)\n",
    "print('nb of total predictions : ', cnt_pred)\n",
    "\n",
    "cnt_golden = []\n",
    "for i,dialog in enumerate(test_data):  \n",
    "    cnt_g = len(dialog['relations'])\n",
    "    cnt = [0] * len(dialog[\"edus\"])\n",
    "    for r in dialog[\"relations\"]:\n",
    "        cnt[r[\"y\"]] += 1\n",
    "    for j in range(len(dialog[\"edus\"])):\n",
    "        if cnt[j] == 0:\n",
    "            cnt_g += 1\n",
    "    cnt_golden += [cnt_g]\n",
    "cnt_golden = sum(cnt_golden)\n",
    "print('nb of relations : ',cnt_golden) \n",
    "\n",
    "cnt_correct = []\n",
    "for i, dialog_pred in enumerate(test_pred):\n",
    "    val = 0\n",
    "    truth = [j[:-1] for j in test_truth[i]]\n",
    "    for pred in dialog_pred : \n",
    "        if tuple(pred) in truth:\n",
    "            val += 1\n",
    "    cnt_correct += [val+1]\n",
    "cnt_correct = sum(list(cnt_correct))\n",
    "print('nb of correct predictions : ', cnt_correct)\n",
    "\n",
    "precision = cnt_correct*1.0/cnt_pred*1.0\n",
    "recall = cnt_correct*1.0/cnt_golden*1.0\n",
    "print('Precition :' , precision, '   Recall : ', recall)\n",
    "print('F1 score : ' , 2*( precision * recall / (precision + recall)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(params.data_path + 'linear_pred_stac.pkl', 'wb') as f:\n",
    "    pickle.dump(predictions, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "345aa2f3f3153332aa0250f3057c4206b814c74c10594863073df44855256fe2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}