object_extraction_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_functions1 import extractor, extractorAurora\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extraction ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "qw_str = \"What is better bread or pizza ?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "will be loaded\n",
      "loading gpu 7\n",
      "loaded extractor\n",
      "will be loaded\n",
      "loading gpu 7\n",
      "extract_objects_predicates gpu 7\n",
      "loaded extractor\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "my_extractor = extractor(model_name = 'bertttt.hdf5', my_device = 7)\n",
    "print (\"loaded extractor\")\n",
    "my_extractor_aurora = extractorAurora(model_name = 'Aurora.hdf5', my_device = 7)\n",
    "print (\"loaded extractor\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "def create_sequence_from_sentence(str_sentences):\n",
    "    return [nltk.word_tokenize(str_sentence) for str_sentence in str_sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in batch_to_ids\n",
      "torch.cuda.device_of -1\n",
      "torch.cuda.device_of -1\n"
     ]
    }
   ],
   "source": [
    "words = create_sequence_from_sentence([qw_str])  \n",
    "tags = my_extractor.model.predict_tags_from_words(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([['What', 'is', 'better', 'bread', 'or', 'pizza', '?']],\n",
       " [['O', 'O', 'B-PREDFULL', 'B-OBJ', 'O', 'B-OBJ', 'O']])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words, tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in batch_to_ids\n",
      "torch.cuda.device_of -1\n",
      "torch.cuda.device_of -1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([['What', 'is', 'better', 'bread', 'or', 'pizza', '?']],\n",
       " [['NONE', 'NONE', 'PRED', 'NONE', 'NONE', 'PROD2', 'NONE']])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = create_sequence_from_sentence([qw_str])  \n",
    "tags = my_extractor_aurora.model.predict_tags_from_words(words)\n",
    "words, tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Or "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in extractor get params 0\n",
      "in batch_to_ids\n",
      "torch.cuda.device_of -1\n",
      "torch.cuda.device_of -1\n",
      "extract_objects_predicates tags ['O', 'O', 'B-PREDFULL', 'B-OBJ', 'O', 'B-OBJ', 'O']\n",
      "extract_objects_predicates words ['What', 'is', 'better', 'bread', 'or', 'pizza', '?']\n",
      "['bread', 'pizza']\n",
      "['better']\n",
      "[]\n",
      "len(objects) 2\n"
     ]
    }
   ],
   "source": [
    "my_extractor.from_string(qw_str)\n",
    "obj1, obj2, pred = my_extractor.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in extractor get params 0\n",
      "in batch_to_ids\n",
      "torch.cuda.device_of -1\n",
      "torch.cuda.device_of -1\n",
      "extract_objects_predicates tags ['NONE', 'NONE', 'PRED', 'NONE', 'NONE', 'PROD2', 'NONE']\n",
      "extract_objects_predicates words ['What', 'is', 'better', 'bread', 'or', 'pizza', '?']\n",
      "[]\n",
      "['better']\n",
      "[]\n",
      "len(objects) 0\n",
      "We try to use spacy\n",
      "We try to use spacy\n",
      "split_sent ['What', 'is', 'better', 'bread', 'or', 'pizza', '?']\n",
      "True\n",
      "comp_elem or\n",
      "tokens ['What', 'is', 'better', 'bread', 'or', 'pizza', '?']\n",
      "comp elem in tokens\n",
      "or simple split_sent 4\n",
      "bread pizza\n"
     ]
    }
   ],
   "source": [
    "my_extractor_aurora.from_string(qw_str)\n",
    "obj1, obj2, pred = my_extractor_aurora.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "obj1, obj2, pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.evaluators.evaluator_base import EvaluatorBase\n",
    "from src.evaluators.evaluator_f1_macro_token_level import EvaluatorF1MacroTokenLevel # choose evaluator type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.classes.datasets_bank import DatasetsBank, DatasetsBankSorted\n",
    "from src.data_io.data_io_connl_ner_2003 import DataIOConnlNer2003\n",
    "\n",
    "data_io = DataIOConnlNer2003()\n",
    "\n",
    "word_sequences_test, tag_sequences_test = data_io.read_data(fn='your_connll_targets.tsv', verbose=True)\n",
    "\n",
    "datasets_bank = DatasetsBank(verbose=True)\n",
    "\n",
    "#datasets_bank.add_train_sequences(word_sequences_train, tag_sequences_train)\n",
    "#datasets_bank.add_dev_sequences(word_sequences_dev, tag_sequences_dev)\n",
    "datasets_bank.add_test_sequences(word_sequences_test, tag_sequences_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_sequences=datasets_bank.word_sequences_test\n",
    "targets_tag_sequences=datasets_bank.tag_sequences_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('your_predicted_tags.txt') as lines:\n",
    "    outputs_tag_sequences = []\n",
    "    for line in lines:\n",
    "        outputs_tag_sequences.append(line.strip().split(', '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_ts = []\n",
    "for line in tag_sequences_test:\n",
    "    for ind, elem in enumerate(line):\n",
    "        if (elem == 'NONE'): \n",
    "            line[ind] = 'O'\n",
    "    new_ts.append(line)\n",
    "tag_sequences_test=new_ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluator = EvaluatorF1MacroTokenLevel()\n",
    "evaluator.get_evaluation_score(tag_sequences_test, outputs_tag_sequences, word_sequences)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}