notebooks/Simple QA End-To-End/Step 3 - Predict Relation and Finish.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Step 3 - Predict Relation and Finish\n",
    "\n",
    "Our goal during this step is to predict the relation and compute the end-to-end accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '../../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lib.utils import get_connection \n",
    "from lib.utils import FB2M_NAME_TABLE\n",
    "from lib.utils import FB2M_KG_TABLE\n",
    "\n",
    "connection = get_connection()\n",
    "cursor = connection.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef7ce87eb4e44e80a618ab76142d9ddf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>end_index</th>\n",
       "      <th>object</th>\n",
       "      <th>predicted_question_tokens</th>\n",
       "      <th>predicted_subject_names</th>\n",
       "      <th>question</th>\n",
       "      <th>question_tokens</th>\n",
       "      <th>relation</th>\n",
       "      <th>start_index</th>\n",
       "      <th>subject</th>\n",
       "      <th>subject_name</th>\n",
       "      <th>subject_name_tokens</th>\n",
       "      <th>candidate_mids</th>\n",
       "      <th>predicted_start_index</th>\n",
       "      <th>predicted_end_index</th>\n",
       "      <th>predicted_subject_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17188</th>\n",
       "      <td>7.0</td>\n",
       "      <td>075s73</td>\n",
       "      <td>[which, town, is, in, new, york, city]</td>\n",
       "      <td>[{'name': 'new york city', 'score': 64.1403045...</td>\n",
       "      <td>which town is in new york city</td>\n",
       "      <td>[which, town, is, in, new, york, city]</td>\n",
       "      <td>location/place_with_neighborhoods/neighborhoods</td>\n",
       "      <td>4.0</td>\n",
       "      <td>02_286</td>\n",
       "      <td>new york city</td>\n",
       "      <td>(new, york, city)</td>\n",
       "      <td>[0y9w2zw, 0mvbtdl, 0lb2n4b, 0dm7wy1, 0mtqpf, 0...</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>new york city</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4793</th>\n",
       "      <td>4.0</td>\n",
       "      <td>017drs</td>\n",
       "      <td>[does, pee, wee, reese, play, shortstop, or, p...</td>\n",
       "      <td>[{'name': 'pee wee reese', 'score': 91.0824661...</td>\n",
       "      <td>does pee wee reese play shortstop or power for...</td>\n",
       "      <td>[does, pee, wee, reese, play, shortstop, or, p...</td>\n",
       "      <td>baseball/baseball_player/position_s</td>\n",
       "      <td>1.0</td>\n",
       "      <td>01bg1k</td>\n",
       "      <td>pee wee reese</td>\n",
       "      <td>(pee, wee, reese)</td>\n",
       "      <td>[01bg1k]</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>pee wee reese</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21187</th>\n",
       "      <td>7.0</td>\n",
       "      <td>0vp3fq</td>\n",
       "      <td>[what, is, a, track, by, lutricia, mcneal, ?]</td>\n",
       "      <td>[{'name': 'lutricia mcneal', 'score': 79.92938...</td>\n",
       "      <td>What is a track by lutricia mcneal?</td>\n",
       "      <td>[what, is, a, track, by, lutricia, mcneal, ?]</td>\n",
       "      <td>music/artist/track</td>\n",
       "      <td>5.0</td>\n",
       "      <td>06tw28</td>\n",
       "      <td>lutricia  mcneal</td>\n",
       "      <td>(lutricia, mcneal)</td>\n",
       "      <td>[0g7gwhy, 06tw28, 03635yk]</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>lutricia mcneal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18730</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0wzyx1</td>\n",
       "      <td>[name, a, recording, by, nelson, mandela]</td>\n",
       "      <td>[{'name': 'nelson mandela', 'score': 57.336029...</td>\n",
       "      <td>Name a recording by nelson mandela</td>\n",
       "      <td>[name, a, recording, by, nelson, mandela]</td>\n",
       "      <td>music/release_track/recording</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0slws_1</td>\n",
       "      <td>nelson mandela</td>\n",
       "      <td>(nelson, mandela)</td>\n",
       "      <td>[0ntz310, 0m6p292, 0_dj598, 0zgdbck, 0l5t41g, ...</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>nelson mandela</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10014</th>\n",
       "      <td>9.0</td>\n",
       "      <td>0crryw4</td>\n",
       "      <td>[name, a, film, in, the, netflix, genre, celti...</td>\n",
       "      <td>[{'name': 'celtic music', 'score': 106.7388153...</td>\n",
       "      <td>Name a film in the netflix genre celtic music.</td>\n",
       "      <td>[name, a, film, in, the, netflix, genre, celti...</td>\n",
       "      <td>media_common/netflix_genre/titles</td>\n",
       "      <td>7.0</td>\n",
       "      <td>01m1y</td>\n",
       "      <td>celtic music</td>\n",
       "      <td>(celtic, music)</td>\n",
       "      <td>[01m1y]</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>celtic music</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       end_index   object                          predicted_question_tokens  \\\n",
       "17188        7.0   075s73             [which, town, is, in, new, york, city]   \n",
       "4793         4.0   017drs  [does, pee, wee, reese, play, shortstop, or, p...   \n",
       "21187        7.0   0vp3fq      [what, is, a, track, by, lutricia, mcneal, ?]   \n",
       "18730        6.0   0wzyx1          [name, a, recording, by, nelson, mandela]   \n",
       "10014        9.0  0crryw4  [name, a, film, in, the, netflix, genre, celti...   \n",
       "\n",
       "                                 predicted_subject_names  \\\n",
       "17188  [{'name': 'new york city', 'score': 64.1403045...   \n",
       "4793   [{'name': 'pee wee reese', 'score': 91.0824661...   \n",
       "21187  [{'name': 'lutricia mcneal', 'score': 79.92938...   \n",
       "18730  [{'name': 'nelson mandela', 'score': 57.336029...   \n",
       "10014  [{'name': 'celtic music', 'score': 106.7388153...   \n",
       "\n",
       "                                                question  \\\n",
       "17188                     which town is in new york city   \n",
       "4793   does pee wee reese play shortstop or power for...   \n",
       "21187                What is a track by lutricia mcneal?   \n",
       "18730                 Name a recording by nelson mandela   \n",
       "10014     Name a film in the netflix genre celtic music.   \n",
       "\n",
       "                                         question_tokens  \\\n",
       "17188             [which, town, is, in, new, york, city]   \n",
       "4793   [does, pee, wee, reese, play, shortstop, or, p...   \n",
       "21187      [what, is, a, track, by, lutricia, mcneal, ?]   \n",
       "18730          [name, a, recording, by, nelson, mandela]   \n",
       "10014  [name, a, film, in, the, netflix, genre, celti...   \n",
       "\n",
       "                                              relation  start_index  subject  \\\n",
       "17188  location/place_with_neighborhoods/neighborhoods          4.0   02_286   \n",
       "4793               baseball/baseball_player/position_s          1.0   01bg1k   \n",
       "21187                               music/artist/track          5.0   06tw28   \n",
       "18730                    music/release_track/recording          4.0  0slws_1   \n",
       "10014                media_common/netflix_genre/titles          7.0    01m1y   \n",
       "\n",
       "           subject_name subject_name_tokens  \\\n",
       "17188     new york city   (new, york, city)   \n",
       "4793      pee wee reese   (pee, wee, reese)   \n",
       "21187  lutricia  mcneal  (lutricia, mcneal)   \n",
       "18730    nelson mandela   (nelson, mandela)   \n",
       "10014      celtic music     (celtic, music)   \n",
       "\n",
       "                                          candidate_mids  \\\n",
       "17188  [0y9w2zw, 0mvbtdl, 0lb2n4b, 0dm7wy1, 0mtqpf, 0...   \n",
       "4793                                            [01bg1k]   \n",
       "21187                         [0g7gwhy, 06tw28, 03635yk]   \n",
       "18730  [0ntz310, 0m6p292, 0_dj598, 0zgdbck, 0l5t41g, ...   \n",
       "10014                                            [01m1y]   \n",
       "\n",
       "       predicted_start_index  predicted_end_index predicted_subject_name  \n",
       "17188                      4                    7          new york city  \n",
       "4793                       1                    4          pee wee reese  \n",
       "21187                      5                    7        lutricia mcneal  \n",
       "18730                      4                    6         nelson mandela  \n",
       "10014                      7                    9           celtic music  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm_notebook\n",
    "from functools import partial\n",
    "\n",
    "tqdm_notebook = partial(tqdm_notebook, leave=False)\n",
    "tqdm_notebook().pandas()\n",
    "\n",
    "df = pd.read_pickle('step_2_generate_candidates.pkl')\n",
    "df[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Facts\n",
    "\n",
    "Given the candidate mids, the we generate candidate facts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "def generate_facts(row):\n",
    "    cursor.execute(\"\"\"SELECT subject_mid, relation, object_mid\n",
    "                      FROM {kg}\n",
    "                      WHERE subject_mid = ANY(%s)\"\"\".format(kg=FB2M_KG_TABLE), (row['candidate_mids'],))\n",
    "    rows = cursor.fetchall()\n",
    "    candidate_facts = defaultdict(lambda: defaultdict(set))\n",
    "    \n",
    "    for subject_mid, relation, object_mid in rows:\n",
    "        candidate_facts[relation][subject_mid].add(object_mid)\n",
    "    \n",
    "    # Convert it back to dict\n",
    "    for relation in candidate_facts:\n",
    "        candidate_facts[relation] = dict(candidate_facts[relation])\n",
    "    candidate_facts = dict(candidate_facts)\n",
    "        \n",
    "    return candidate_facts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8973d4cc89745d3b903f2f7b6b6dfd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "df['candidate_facts'] = df.progress_apply(generate_facts, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Number of Relations: 18.669663853921705\n"
     ]
    }
   ],
   "source": [
    "print('Average Number of Relations:', sum(len(r) for r in df['candidate_facts']) / df.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upperbounds\n",
    "\n",
    "Check the accuracy of the candidates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7284fb564fb47d0b3320b51da9d91d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Object Canditate Accuracy: 0.9535205422603403\n",
      "Relation Canditate Accuracy: 0.9673537142066676\n",
      "Subject Canditate Accuracy: 0.9535205422603403\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "\n",
    "correct_object = 0\n",
    "correct_relation = 0\n",
    "correct_subject = 0\n",
    "for index, row in tqdm_notebook(df.iterrows(), total=df.shape[0]):\n",
    "    facts = row['candidate_facts']\n",
    "\n",
    "    if row['relation'] in facts:\n",
    "        correct_relation += 1\n",
    "        if row['subject'] in facts[row['relation']]:\n",
    "            correct_subject += 1\n",
    "            if row['object'] in facts[row['relation']][row['subject']]:\n",
    "                correct_object += 1    \n",
    "        \n",
    "\n",
    "print('Object Canditate Accuracy:', correct_object / df.shape[0])\n",
    "print('Relation Canditate Accuracy:', correct_relation / df.shape[0])\n",
    "print('Subject Canditate Accuracy:', correct_subject / df.shape[0])\n",
    "\n",
    "# Object Canditate Accuracy: 0.9566620562471185\n",
    "# Relation Canditate Accuracy: 0.9678192715537114\n",
    "# Subject Canditate Accuracy: 0.9566620562471185"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models\n",
    "\n",
    "Load models to be used in our final step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lib.checkpoint import Checkpoint\n",
    "\n",
    "def load_checkpoint(path):\n",
    "    checkpoint = Checkpoint(checkpoint_path=path, device=0)\n",
    "    checkpoint.model.train(mode=False)\n",
    "    checkpoint.model.apply(lambda m: m.flatten_parameters() if hasattr(m, 'flatten_parameters') else None)\n",
    "    return checkpoint\n",
    "\n",
    "cuda = lambda v: v.cuda() if torch.cuda.is_available() else t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15317be55d3c452ebe92fa7aac084b65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>end_index</th>\n",
       "      <th>object</th>\n",
       "      <th>predicted_question_tokens</th>\n",
       "      <th>predicted_subject_names</th>\n",
       "      <th>question</th>\n",
       "      <th>question_tokens</th>\n",
       "      <th>relation</th>\n",
       "      <th>start_index</th>\n",
       "      <th>subject</th>\n",
       "      <th>subject_name</th>\n",
       "      <th>subject_name_tokens</th>\n",
       "      <th>candidate_mids</th>\n",
       "      <th>predicted_start_index</th>\n",
       "      <th>predicted_end_index</th>\n",
       "      <th>predicted_subject_name</th>\n",
       "      <th>candidate_facts</th>\n",
       "      <th>predicted_predicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17188</th>\n",
       "      <td>7.0</td>\n",
       "      <td>075s73</td>\n",
       "      <td>[which, town, is, in, new, york, city]</td>\n",
       "      <td>[{'name': 'new york city', 'score': 64.1403045...</td>\n",
       "      <td>which town is in new york city</td>\n",
       "      <td>[which, town, is, in, new, york, city]</td>\n",
       "      <td>location/place_with_neighborhoods/neighborhoods</td>\n",
       "      <td>4.0</td>\n",
       "      <td>02_286</td>\n",
       "      <td>new york city</td>\n",
       "      <td>(new, york, city)</td>\n",
       "      <td>[0y9w2zw, 0mvbtdl, 0lb2n4b, 0dm7wy1, 0mtqpf, 0...</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>new york city</td>\n",
       "      <td>{'music/recording/tracks': {'013nwy7': {'0ltpb...</td>\n",
       "      <td>which town is in &lt;e&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4793</th>\n",
       "      <td>4.0</td>\n",
       "      <td>017drs</td>\n",
       "      <td>[does, pee, wee, reese, play, shortstop, or, p...</td>\n",
       "      <td>[{'name': 'pee wee reese', 'score': 91.0824661...</td>\n",
       "      <td>does pee wee reese play shortstop or power for...</td>\n",
       "      <td>[does, pee, wee, reese, play, shortstop, or, p...</td>\n",
       "      <td>baseball/baseball_player/position_s</td>\n",
       "      <td>1.0</td>\n",
       "      <td>01bg1k</td>\n",
       "      <td>pee wee reese</td>\n",
       "      <td>(pee, wee, reese)</td>\n",
       "      <td>[01bg1k]</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>pee wee reese</td>\n",
       "      <td>{'people/person/nationality': {'01bg1k': {'09c...</td>\n",
       "      <td>does &lt;e&gt; play shortstop or power forward</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21187</th>\n",
       "      <td>7.0</td>\n",
       "      <td>0vp3fq</td>\n",
       "      <td>[what, is, a, track, by, lutricia, mcneal, ?]</td>\n",
       "      <td>[{'name': 'lutricia mcneal', 'score': 79.92938...</td>\n",
       "      <td>What is a track by lutricia mcneal?</td>\n",
       "      <td>[what, is, a, track, by, lutricia, mcneal, ?]</td>\n",
       "      <td>music/artist/track</td>\n",
       "      <td>5.0</td>\n",
       "      <td>06tw28</td>\n",
       "      <td>lutricia  mcneal</td>\n",
       "      <td>(lutricia, mcneal)</td>\n",
       "      <td>[0g7gwhy, 06tw28, 03635yk]</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>lutricia mcneal</td>\n",
       "      <td>{'music/release/track': {'0g7gwhy': {'0v180y',...</td>\n",
       "      <td>what is a track by &lt;e&gt; ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18730</th>\n",
       "      <td>6.0</td>\n",
       "      <td>0wzyx1</td>\n",
       "      <td>[name, a, recording, by, nelson, mandela]</td>\n",
       "      <td>[{'name': 'nelson mandela', 'score': 57.336029...</td>\n",
       "      <td>Name a recording by nelson mandela</td>\n",
       "      <td>[name, a, recording, by, nelson, mandela]</td>\n",
       "      <td>music/release_track/recording</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0slws_1</td>\n",
       "      <td>nelson mandela</td>\n",
       "      <td>(nelson, mandela)</td>\n",
       "      <td>[0ntz310, 0m6p292, 0_dj598, 0zgdbck, 0l5t41g, ...</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>nelson mandela</td>\n",
       "      <td>{'music/release_track/recording': {'0nqjllh': ...</td>\n",
       "      <td>name a recording by &lt;e&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10014</th>\n",
       "      <td>9.0</td>\n",
       "      <td>0crryw4</td>\n",
       "      <td>[name, a, film, in, the, netflix, genre, celti...</td>\n",
       "      <td>[{'name': 'celtic music', 'score': 106.7388153...</td>\n",
       "      <td>Name a film in the netflix genre celtic music.</td>\n",
       "      <td>[name, a, film, in, the, netflix, genre, celti...</td>\n",
       "      <td>media_common/netflix_genre/titles</td>\n",
       "      <td>7.0</td>\n",
       "      <td>01m1y</td>\n",
       "      <td>celtic music</td>\n",
       "      <td>(celtic, music)</td>\n",
       "      <td>[01m1y]</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>celtic music</td>\n",
       "      <td>{'media_common/netflix_genre/titles': {'01m1y'...</td>\n",
       "      <td>name a film in the netflix genre &lt;e&gt; .</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       end_index   object                          predicted_question_tokens  \\\n",
       "17188        7.0   075s73             [which, town, is, in, new, york, city]   \n",
       "4793         4.0   017drs  [does, pee, wee, reese, play, shortstop, or, p...   \n",
       "21187        7.0   0vp3fq      [what, is, a, track, by, lutricia, mcneal, ?]   \n",
       "18730        6.0   0wzyx1          [name, a, recording, by, nelson, mandela]   \n",
       "10014        9.0  0crryw4  [name, a, film, in, the, netflix, genre, celti...   \n",
       "\n",
       "                                 predicted_subject_names  \\\n",
       "17188  [{'name': 'new york city', 'score': 64.1403045...   \n",
       "4793   [{'name': 'pee wee reese', 'score': 91.0824661...   \n",
       "21187  [{'name': 'lutricia mcneal', 'score': 79.92938...   \n",
       "18730  [{'name': 'nelson mandela', 'score': 57.336029...   \n",
       "10014  [{'name': 'celtic music', 'score': 106.7388153...   \n",
       "\n",
       "                                                question  \\\n",
       "17188                     which town is in new york city   \n",
       "4793   does pee wee reese play shortstop or power for...   \n",
       "21187                What is a track by lutricia mcneal?   \n",
       "18730                 Name a recording by nelson mandela   \n",
       "10014     Name a film in the netflix genre celtic music.   \n",
       "\n",
       "                                         question_tokens  \\\n",
       "17188             [which, town, is, in, new, york, city]   \n",
       "4793   [does, pee, wee, reese, play, shortstop, or, p...   \n",
       "21187      [what, is, a, track, by, lutricia, mcneal, ?]   \n",
       "18730          [name, a, recording, by, nelson, mandela]   \n",
       "10014  [name, a, film, in, the, netflix, genre, celti...   \n",
       "\n",
       "                                              relation  start_index  subject  \\\n",
       "17188  location/place_with_neighborhoods/neighborhoods          4.0   02_286   \n",
       "4793               baseball/baseball_player/position_s          1.0   01bg1k   \n",
       "21187                               music/artist/track          5.0   06tw28   \n",
       "18730                    music/release_track/recording          4.0  0slws_1   \n",
       "10014                media_common/netflix_genre/titles          7.0    01m1y   \n",
       "\n",
       "           subject_name subject_name_tokens  \\\n",
       "17188     new york city   (new, york, city)   \n",
       "4793      pee wee reese   (pee, wee, reese)   \n",
       "21187  lutricia  mcneal  (lutricia, mcneal)   \n",
       "18730    nelson mandela   (nelson, mandela)   \n",
       "10014      celtic music     (celtic, music)   \n",
       "\n",
       "                                          candidate_mids  \\\n",
       "17188  [0y9w2zw, 0mvbtdl, 0lb2n4b, 0dm7wy1, 0mtqpf, 0...   \n",
       "4793                                            [01bg1k]   \n",
       "21187                         [0g7gwhy, 06tw28, 03635yk]   \n",
       "18730  [0ntz310, 0m6p292, 0_dj598, 0zgdbck, 0l5t41g, ...   \n",
       "10014                                            [01m1y]   \n",
       "\n",
       "       predicted_start_index  predicted_end_index predicted_subject_name  \\\n",
       "17188                      4                    7          new york city   \n",
       "4793                       1                    4          pee wee reese   \n",
       "21187                      5                    7        lutricia mcneal   \n",
       "18730                      4                    6         nelson mandela   \n",
       "10014                      7                    9           celtic music   \n",
       "\n",
       "                                         candidate_facts  \\\n",
       "17188  {'music/recording/tracks': {'013nwy7': {'0ltpb...   \n",
       "4793   {'people/person/nationality': {'01bg1k': {'09c...   \n",
       "21187  {'music/release/track': {'0g7gwhy': {'0v180y',...   \n",
       "18730  {'music/release_track/recording': {'0nqjllh': ...   \n",
       "10014  {'media_common/netflix_genre/titles': {'01m1y'...   \n",
       "\n",
       "                            predicted_predicate  \n",
       "17188                      which town is in <e>  \n",
       "4793   does <e> play shortstop or power forward  \n",
       "21187                  what is a track by <e> ?  \n",
       "18730                   name a recording by <e>  \n",
       "10014    name a film in the netflix genre <e> .  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_predicted_predicate(row):\n",
    "    formatted_question =  ''\n",
    "    for i, token in enumerate(row['predicted_question_tokens']):\n",
    "        if i == row['predicted_start_index']:\n",
    "            formatted_question += '<e>'\n",
    "        elif i > row['predicted_start_index'] and i < row['predicted_end_index']:\n",
    "            continue\n",
    "        else:\n",
    "            formatted_question += token.lower().strip()\n",
    "        formatted_question += ' '\n",
    "    formatted_question = formatted_question.strip()\n",
    "    return formatted_question\n",
    "\n",
    "df['predicted_predicate'] = df.progress_apply(get_predicted_predicate, axis=1)\n",
    "df[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Softmax Relation Model\n",
    "\n",
    "Load the softmax relation model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: where was <e> born?\n",
      "Scores:\n",
      "[0.9917193651199341, 0.00010895365267060697]\n"
     ]
    }
   ],
   "source": [
    "from torch.autograd import Variable\n",
    "import math\n",
    "import torch\n",
    "\n",
    "def get_softmax_relation_score(checkpoint, question, relations):\n",
    "    relations = [r for r in relations]\n",
    "    relations = [checkpoint.relation_encoder.encode(r)[0] for r in relations]\n",
    "    mask = set(relations)\n",
    "    mask = [1 if i in relations else 0 for i in range(checkpoint.relation_encoder.vocab_size)]\n",
    "    mask = cuda(Variable(torch.FloatTensor(mask), volatile=True))\n",
    "    question = checkpoint.text_encoder.encode(question)\n",
    "    question = cuda(Variable(torch.LongTensor(question).unsqueeze(1), volatile=True))\n",
    "    output_batch = checkpoint.model(question, mask).exp_().data\n",
    "    output_batch = output_batch.squeeze(0)\n",
    "    return [output_batch[r] for r in relations]\n",
    "    \n",
    "## Test ##\n",
    "# Checkpoint with 0.880\n",
    "checkpoint = load_checkpoint('../../pretrained_models/relation_classifier.02_02_07:59:28/1746.pt')\n",
    "question = 'where was <e> born?'\n",
    "print('Question:', question)\n",
    "print('Scores:')\n",
    "print(get_softmax_relation_score(checkpoint, question, ['people/person/place_of_birth',\n",
    "                                                        'location/location/people_born_here']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56c0fb768f884f4c97735415491b25e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "checkpoint = load_checkpoint('../../pretrained_models/relation_classifier.02_02_13:31:11/189.pt')\n",
    "def add_softmax_scores(row):\n",
    "    if len(row['candidate_facts']) != 0:\n",
    "        candidate_relations = list(row['candidate_facts'].keys())\n",
    "        return get_softmax_relation_score(checkpoint, row['predicted_predicate'], candidate_relations)\n",
    "    else:\n",
    "        return None\n",
    "    \n",
    "df['softmax_scores'] = df.progress_apply(add_softmax_scores, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1635257bfbd4506928cfe49e24c99fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28b44e1989d746b8839b03da83ecdbf5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from functools import partial\n",
    "\n",
    "def add_softmax_ensemble_scores(row, checkpoint):\n",
    "    if len(row['candidate_facts']) != 0:\n",
    "        candidate_relations = list(row['candidate_facts'].keys())\n",
    "        scores = get_softmax_relation_score(checkpoint, row['predicted_predicate'], candidate_relations)\n",
    "        # Sum together...\n",
    "        return [ensemble_score + score for ensemble_score, score in zip(row['softmax_ensemble_scores'], scores)]\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "# Copy previous single model scores\n",
    "df['softmax_ensemble_scores'] = df['softmax_scores']\n",
    "ensemble = [\n",
    "#     '../../pretrained_models/relation_classifier.02_02_13:31:11/189.pt', # 1212212=seed, 88.38%=dev_accuracy\n",
    "    '../../pretrained_models/relation_classifier.02_02_13:23:02/223.pt', # 457=seed, 88.26%=dev_accuracy\n",
    "    '../../pretrained_models/relation_classifier.02_02_07:59:28/1746.pt', # 123=seed, 88.25%=dev_accuracy\n",
    "]\n",
    "\n",
    "for path in ensemble:\n",
    "    checkpoint = load_checkpoint(path)\n",
    "    df['softmax_ensemble_scores'] = df.progress_apply(partial(add_softmax_ensemble_scores,\n",
    "                                                              checkpoint=checkpoint), axis=1)\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final End-To-End Metric\n",
    "\n",
    "Given candidate facts compute the end-to-end metric. Below we start a couple code blocks with basic utilities used by each version."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### FB2M Probability to Train Probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "cursor.execute('SELECT relation, count(*) FROM ' + FB2M_KG_TABLE + ' GROUP BY relation')\n",
    "relation_count_fb2m = {}\n",
    "for relation, n_rows_fb2m in cursor.fetchall():\n",
    "    relation_count_fb2m[relation] = n_rows_fb2m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "from lib.simple_qa import load_simple_qa \n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "df_train, = load_simple_qa(train=True)\n",
    "\n",
    "total_rows_train = df_train.shape[0]\n",
    "cursor.execute('SELECT count(*) FROM ' + FB2M_KG_TABLE)\n",
    "total_rows_fb_two_kg = cursor.fetchone()[0]\n",
    "\n",
    "# Given we see a relation occuring with %x probability in KG, we use `transform_probability_from_kg_to_train`\n",
    "# to get the probability of the relation occuring in SimpleQuestions.\n",
    "transform_probability_from_kg_to_train = defaultdict(int)\n",
    "for relation, n_rows_train in df_train.relation.value_counts().iteritems():\n",
    "    n_facts_fb2m = relation_count_fb2m[relation]\n",
    "    \n",
    "    relation_probability_kg = n_facts_fb2m / total_rows_fb_two_kg\n",
    "    relation_probability_train = n_rows_train / total_rows_train\n",
    "    \n",
    "    transform_probability_from_kg_to_train[relation] = relation_probability_train / relation_probability_kg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluation\n",
    "\n",
    "Question interpretations allow us to allow questions to be interpreted multiple times."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "question_interpretations = json.load(open('../Simple QA Numbers/question_interpretations.txt'))\n",
    "question_interpretations = [set([tuple(pair) for pair in row]) for row in question_interpretations]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the predicate row for every training example. We use this to analyze the closest negative predicate to confirm if the data supports the false relation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from ../../notebooks/Simple QA Numbers/HYPOTHESIS - Question Refers to Multiple Subjects.ipynb\n",
      "importing Jupyter notebook from ../../notebooks/Simple QA Numbers/HYPOTHESIS - Subject Name not in Question.ipynb\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02a2816eb3c0464ab122609e8ca97e59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=75910), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "importing Jupyter notebook from ../../notebooks/Simple QA Models/Subject Recognition Data.ipynb\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6520c4c3590e4bdab47af5ef2c4e530f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=75910), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from functools import partial\n",
    "import importlib\n",
    "import lib.import_notebook\n",
    "\n",
    "edit_distance_link_alias = importlib.import_module(\n",
    "                \"notebooks.Simple QA Numbers.HYPOTHESIS - Question Refers to Multiple Subjects\").edit_distance_link_alias\n",
    "normalize = importlib.import_module(\n",
    "                \"notebooks.Simple QA Numbers.HYPOTHESIS - Subject Name not in Question\").normalize\n",
    "\n",
    "# Create a column with the subject_name linked per example\n",
    "df_train['subject_name'] = df_train.progress_apply(partial(edit_distance_link_alias, cursor, normalize), axis=1)\n",
    "\n",
    "find_subject_name_span = importlib.import_module(\n",
    "                \"notebooks.Simple QA Models.Subject Recognition Data\").find_subject_name_span\n",
    "\n",
    "df_train = df_train.progress_apply(find_subject_name_span, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d06fd5d3eec3475086fcd70f9d7da68d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=75910), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>end_index</th>\n",
       "      <th>object</th>\n",
       "      <th>question</th>\n",
       "      <th>question_tokens</th>\n",
       "      <th>relation</th>\n",
       "      <th>start_index</th>\n",
       "      <th>subject</th>\n",
       "      <th>subject_name</th>\n",
       "      <th>subject_name_tokens</th>\n",
       "      <th>predicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>01cj3p</td>\n",
       "      <td>what is the book e about</td>\n",
       "      <td>[what, is, the, book, e, about]</td>\n",
       "      <td>book/written_work/subjects</td>\n",
       "      <td>4.0</td>\n",
       "      <td>04whkz5</td>\n",
       "      <td>e</td>\n",
       "      <td>(e,)</td>\n",
       "      <td>what is the book &lt;e&gt; about</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.0</td>\n",
       "      <td>0sjc7c1</td>\n",
       "      <td>to what release does the release track cardiac...</td>\n",
       "      <td>[to, what, release, does, the, release, track,...</td>\n",
       "      <td>music/release_track/release</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0tp2p24</td>\n",
       "      <td>cardiac arrest</td>\n",
       "      <td>(cardiac, arrest)</td>\n",
       "      <td>to what release does the release track &lt;e&gt; com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.0</td>\n",
       "      <td>07ssc</td>\n",
       "      <td>what country was the film the debt from</td>\n",
       "      <td>[what, country, was, the, film, the, debt, from]</td>\n",
       "      <td>film/film/country</td>\n",
       "      <td>5.0</td>\n",
       "      <td>04j0t75</td>\n",
       "      <td>the debt</td>\n",
       "      <td>(the, debt)</td>\n",
       "      <td>what country was the film &lt;e&gt; from</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.0</td>\n",
       "      <td>0p600l</td>\n",
       "      <td>what songs have nobuo uematsu produced?</td>\n",
       "      <td>[what, songs, have, nobuo, uematsu, produced, ?]</td>\n",
       "      <td>music/producer/tracks_produced</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0ftqr</td>\n",
       "      <td>nobuo uematsu</td>\n",
       "      <td>(nobuo, uematsu)</td>\n",
       "      <td>what songs have &lt;e&gt; produced ?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>0677ng</td>\n",
       "      <td>Who produced eve-olution?</td>\n",
       "      <td>[who, produced, eve, -, olution, ?]</td>\n",
       "      <td>music/release/producers</td>\n",
       "      <td>2.0</td>\n",
       "      <td>036p007</td>\n",
       "      <td>eve-olution</td>\n",
       "      <td>(eve, -, olution)</td>\n",
       "      <td>who produced &lt;e&gt; ?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   end_index   object                                           question  \\\n",
       "0        5.0   01cj3p                           what is the book e about   \n",
       "1        9.0  0sjc7c1  to what release does the release track cardiac...   \n",
       "2        7.0    07ssc            what country was the film the debt from   \n",
       "3        5.0   0p600l            what songs have nobuo uematsu produced?   \n",
       "4        5.0   0677ng                          Who produced eve-olution?   \n",
       "\n",
       "                                     question_tokens  \\\n",
       "0                    [what, is, the, book, e, about]   \n",
       "1  [to, what, release, does, the, release, track,...   \n",
       "2   [what, country, was, the, film, the, debt, from]   \n",
       "3   [what, songs, have, nobuo, uematsu, produced, ?]   \n",
       "4                [who, produced, eve, -, olution, ?]   \n",
       "\n",
       "                         relation  start_index  subject    subject_name  \\\n",
       "0      book/written_work/subjects          4.0  04whkz5               e   \n",
       "1     music/release_track/release          7.0  0tp2p24  cardiac arrest   \n",
       "2               film/film/country          5.0  04j0t75        the debt   \n",
       "3  music/producer/tracks_produced          3.0    0ftqr   nobuo uematsu   \n",
       "4         music/release/producers          2.0  036p007     eve-olution   \n",
       "\n",
       "  subject_name_tokens                                          predicate  \n",
       "0                (e,)                         what is the book <e> about  \n",
       "1   (cardiac, arrest)  to what release does the release track <e> com...  \n",
       "2         (the, debt)                 what country was the film <e> from  \n",
       "3    (nobuo, uematsu)                     what songs have <e> produced ?  \n",
       "4   (eve, -, olution)                                 who produced <e> ?  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_predicate(row):\n",
    "    if not isinstance(row['question_tokens'], list):\n",
    "        return ''\n",
    "        \n",
    "    formatted_question =  ''\n",
    "    for i, token in enumerate(row['question_tokens']):\n",
    "        if i == row['start_index']:\n",
    "            formatted_question += '<e>'\n",
    "        elif i > row['start_index'] and i < row['end_index']:\n",
    "            continue\n",
    "        else:\n",
    "            formatted_question += token.lower().strip()\n",
    "        formatted_question += ' '\n",
    "    formatted_question = formatted_question.strip()\n",
    "    return formatted_question\n",
    "\n",
    "df_train['predicate'] = df_train.progress_apply(get_predicate, axis=1)\n",
    "df_train[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation function with negative sampling for error analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lib.utils import format_pipe_table\n",
    "from Levenshtein import distance\n",
    "from heapq import nsmallest\n",
    "\n",
    "relation_count_train = df_train.relation.value_counts().to_dict()\n",
    "\n",
    "def evaluate(predicted):\n",
    "    subject_and_relation_correct = 0 # Official metric from the Simple Questions dataset\n",
    "    object_correct = 0\n",
    "    relation_correct = 0\n",
    "    subject_correct = 0\n",
    "    correct_interpretation = 0\n",
    "    no_interpretation = 0\n",
    "    negative_samples = []\n",
    "    \n",
    "    for i, (_, row) in tqdm_notebook(enumerate(df.iterrows()), total=df.shape[0]):\n",
    "        relation, subject, objects = predicted[i]\n",
    "        if tuple([relation, subject]) in question_interpretations[i]:\n",
    "            correct_interpretation += 1\n",
    "        elif len(question_interpretations[i]) != 0:\n",
    "            count = relation_count_train[row['relation']] if row['relation'] in relation_count_train else 0\n",
    "            false_predicates = df_train[df_train['relation'] == relation]['predicate']\n",
    "            if len(false_predicates) > 0:\n",
    "                closest_false_predicate = nsmallest(3, false_predicates,\n",
    "                    key=lambda p: distance(p, row['predicted_predicate']))\n",
    "            negative_samples.append({\n",
    "                'Question': row['question'],\n",
    "                'Closest False Predicate': closest_false_predicate,\n",
    "                'False Relation': relation,\n",
    "                'True Relation': row['relation'],\n",
    "                'True Relation Count': count,\n",
    "                'Predicted Subject Name': row['predicted_subject_name'],\n",
    "                'Subject Name': row['subject_name'],\n",
    "            })\n",
    "        else:\n",
    "            no_interpretation += 1\n",
    "            \n",
    "        if relation == row['relation'] and subject == row['subject']:\n",
    "            subject_and_relation_correct += 1\n",
    "        else:\n",
    "            if relation and relation in row['candidate_facts']:\n",
    "                subjects = list(row['candidate_facts'][relation].keys())[:3]\n",
    "                cursor.execute('SELECT mid, alias FROM fb_two_name WHERE mid = ANY(%s)', (subjects,))\n",
    "                subject_aliases = list(cursor.fetchall())[:3]\n",
    "            else:\n",
    "                subjects = None\n",
    "                subject_aliases = None\n",
    "        if objects and row['object'] in objects:\n",
    "            object_correct += 1\n",
    "        if relation == row['relation']:\n",
    "            relation_correct += 1\n",
    "        if subject == row['subject']:\n",
    "            subject_correct += 1\n",
    "        \n",
    "    print('Subject & Relation Accuracy (SOTA 78.7%%): %f [%d of %d]' %\n",
    "          (subject_and_relation_correct / df.shape[0], subject_and_relation_correct, df.shape[0]))\n",
    "    print('Object Accuracy: %f [%d of %d]' %\n",
    "              (object_correct / df.shape[0], object_correct, df.shape[0]))\n",
    "    print('Relation Accuracy (SOTA 88.4%%): %f [%d of %d]' %\n",
    "              (relation_correct / df.shape[0], relation_correct, df.shape[0]))\n",
    "    print('Subject Accuracy (SOTA 79%%): %f [%d of %d]' %\n",
    "              (subject_correct / df.shape[0], subject_correct, df.shape[0]))\n",
    "    # Is this a valid interpretation of the question\n",
    "    print('Correct Interpretation: %f [%d of %d]' %\n",
    "              (correct_interpretation / df.shape[0], correct_interpretation, df.shape[0]))\n",
    "    print('No Interpretation: %f [%d of %d]' %\n",
    "              (no_interpretation / df.shape[0], no_interpretation, df.shape[0]))\n",
    "    print('Negative Sample:')\n",
    "    print(format_pipe_table(negative_samples[:50], columns=['Bucket', 'Closest False Predicate', 'True Relation Count', 'Question', 'True Relation',\n",
    "                                                            'False Relation', 'Subject Name', 'Predicted Subject Name']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Version: Softmax\n",
    "\n",
    "For the version 0 of softmax, we use the highest predicted relation with not additional considerations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b82e9db93254f8489c126b137bc7b4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7cbc555f15941d7ab39c3becfa93a45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Subject & Relation Accuracy (SOTA 78.7%): 0.760133 [16485 of 21687]\n",
      "Object Accuracy: 0.811131 [17591 of 21687]\n",
      "Relation Accuracy (SOTA 88.4%): 0.875778 [18993 of 21687]\n",
      "Subject Accuracy (SOTA 79%): 0.813298 [17638 of 21687]\n",
      "Correct Interpretation: 0.912436 [19788 of 21687]\n",
      "No Interpretation: 0.019413 [421 of 21687]\n",
      "Negative Sample:\n",
      "| Index | Bucket | Closest False Predicate | True Relation Count | Question | True Relation | False Relation | Subject Name | Predicted Subject Name |\n",
      "| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n",
      "| 0 | nan | what town is in <e> | 39 | which town is in new york city | location/place_with_neighborhoods/neighborhoods | location/location/contains | new york city | new york city |\n",
      "| 1 | nan | what is a city in <e> , illinois | 1243 | what is a city in braxton county, west virginia | location/location/contains | location/us_county/hud_county_place | braxton county | braxton county |\n",
      "| 2 | nan | what is the license for <e> | 30 | whats the name of a law enforcement agency | organization/organization_sector/organizations_in_this_sector | computer/software/license | law enforcement agency | enforcement |\n",
      "| 3 | nan | what is a notable type of <e> ? | 1243 | What is an island located in united states minor outlying islands? | location/location/contains | common/topic/notable_types | united states minor outlying islands | outlying islands |\n",
      "| 4 | nan | in which country was <e> album released | 761 | in which country was the greatest hits album released | music/release/region | music/release/region | greatest hits | the greatest hits |\n",
      "| 5 | nan | what is an album recorded by <e> ? | 1044 | jackie is an album recorded by what artist? | music/album/artist | music/artist/album | jackie | jackie |\n",
      "| 6 | nan | what is <e> 's profession ? | 15 | What is cesky terrier's temperance? | biology/animal_breed/temperament | common/topic/notable_types | cesky terrier | cesky terrier |\n",
      "| 7 | nan | what is a live <e> by archer prewitt ? | 656 | what is the cover album by Japanese pop singer Ken Hirai | music/artist/album | music/album_release_type/albums | ken hirai | album |\n",
      "| 8 | nan | what is the area code for <e> | 0 | what is the parent institution for cohn-haddow center for judaic studies | base/jewlib/research_center/parent_institution | location/location/containedby | cohn-haddow center for judaic studies | cohn - haddow center for judaic studies |\n",
      "| 9 | nan | where is <e> from | 5 | where is produced by closer | theater/theater_production/performed_at | film/film/country | closer | closer |\n",
      "| 10 | nan | which state is <e> located in | 2472 | Which country is kenai fjords national park located in | location/location/containedby | base/usnationalparks/us_national_park/state | kenai fjords national park | kenai fjords national park |\n",
      "| 11 | nan | who was born in the <e> ? | 37 | What is a color of the st helens rlfc | sports/sports_team/colors | location/location/people_born_here | st helens rfc | st helens |\n",
      "| 12 | nan | which city is in <e> | 16 | which city is river dee, aberdeenshire in  | geography/river/cities | location/location/contains | river dee | dee , aberdeenshire |\n",
      "| 13 | nan | what country is <e> in | 1 | which city is in bosnian county | base/aareas/schema/administrative_area_type/subdivides_place | base/aareas/schema/administrative_area_type/pertains_to | bosnian county | bosnian county |\n",
      "| 14 | nan | what classification is <e> | 159 | what classification is the drug sodium sulfacetamide and sulfur 0.05 kit | medicine/drug_formulation/legal_status | medicine/drug_formulation/drug_category | sodium sulfacetamide and sulfur 0.05 kit | sodium sulfacetamide and sulfur 0.05 kit |\n",
      "| 15 | nan | what is <e> 's gender identity | 0 | what is mandy bright's sexual orientation  | base/adultentertainment/adult_entertainer/sexual_orientation | people/person/gender | mandy | mandy bright |\n",
      "| 16 | nan | which releases has <e> as a track ? | 646 | which releases have gold as the track? | music/recording/releases | music/release_track/release | gold | gold |\n",
      "| 17 | nan | what movie was directed by <e> | 1070 | what thriller was directed by Félix Enríquez Alcalá | film/film_genre/films_in_this_genre | film/director/film | thriller | felix enriquez alcala |\n",
      "| 18 | nan | what is the type of album that <e> is ? | 19 | what's the first episode of the fifth season of deja vu (the outer limits) | tv/tv_series_episode/next_episode | music/album/album_content_type | deja vu | the fifth season of |\n",
      "| 19 | nan | where did the film <e> originate ? | 441 | Where is the artist the lab originate  | music/artist/origin | film/film/country | the lab | the lab |\n",
      "| 20 | nan | what color is <e> | 7 | What color of horse is george washington | base/thoroughbredracing/thoroughbred_racehorse/color | education/educational_institution/colors | george washington | george washington |\n",
      "| 21 | nan | the song <e> is from which album | 676 | die in your arms is the single from which album  | music/release_track/release | music/release_track/recording | die in your arms | die in your arms |\n",
      "| 22 | nan | what is the title of a film with the <e> genre ? | 1070 | What is the drama film where black-and-white technique was used? | film/film_genre/films_in_this_genre | film/film_genre/films_in_this_genre | black-and-white | drama film |\n",
      "| 23 | nan | what entity governs the <e> | 123 | what American politician gave her name for the eleanor roosevelt national historic site | symbols/namesake/named_after | protected_sites/protected_site/governing_body | eleanor roosevelt national historic site | eleanor roosevelt national historic |\n",
      "| 24 | nan | what is <e> 's profession ? | 847 | What is aninuddha roy chowdhury's profession? | people/person/profession | common/topic/notable_types | aniruddha roy chowdhury | chowdhury |\n",
      "| 25 | nan | which genre does <e> fall under | 960 | What genre does casino royale fall under? | media_common/netflix_title/netflix_genres | book/book/genre | casino royale | casino royale |\n",
      "| 26 | nan | what fires were started as a result of <e> | 1 | what scandinavian airlines flight was involved in an accident | aviation/airline/accidents | base/fires/fire_cause/fires_caused_this_way | scandinavian airlines | accident |\n",
      "| 27 | nan | what mountains are located in the <e> | 1243 | what mountains are located within white mountains? | location/location/contains | geography/mountain_range/mountains | white mountains | white mountains |\n",
      "| 28 | nan | what genre is <e> video game ? | 169 | What is an adventure cvg game? | cvg/cvg_genre/games | cvg/computer_videogame/cvg_genre | adeventure | adventure |\n",
      "| 29 | nan | name a film that was shown at the <e> | 19 | What's a movie that was shown at the 2010 fantasia festival | film/film_festival_event/films | film/film_festival_event/films | 2010 fantasia film festival | 2010 fantasia festival |\n",
      "| 30 | nan | name an <e> person | 1070 | name a 2007 American film parody  | film/film_genre/films_in_this_genre | people/ethnicity/people | parody | american |\n",
      "| 31 | nan | which german city is <e> from | 1 | which department contains caroline bynum | education/academic/departments_old | people/person/place_of_birth | caroline bynum | caroline bynum |\n",
      "| 32 | nan | what 's one of the songs on the album <e> | 411 | What is one of the songs on the free album? | music/release_track/recording | music/release/track | free | free |\n",
      "| 33 | nan | is <e> pop music or folk | 1553 | is jane child pop music or j-pop | music/artist/genre | music/artist/genre | jane child | jane |\n",
      "| 34 | nan | who is the featured artist on the album <e> ? | 7 | what is the tittle of an entrepreneur who opens and runs restaurants professionally | people/profession/specializations | music/album/featured_artists | entrepreneur | opens |\n",
      "| 35 | nan | what is the country code for <e> | 1 | which country locates japanese city | base/aareas/schema/administrative_area_type/iso_country | base/aareas/schema/administrative_area_type/expected_codes | japanese city | japanese city |\n",
      "| 36 | nan | which biological process is <e> a part of | 0 | What kind of gas does a nitrogen fixation produce | base/process/process/output | base/change/process/part_of_process | nitrogen fixation | nitrogen fixation |\n",
      "| 37 | nan | what type of game is <e> | 226 | what type of game is the ncaa football 07 | cvg/computer_videogame/gameplay_modes | cvg/computer_videogame/cvg_genre | ncaa football 07 | ncaa football 07 |\n",
      "| 38 | nan | what college can be found on <e> | 18 | what college can be accessed by flying to the london heathrow airport | aviation/airport/serves | location/location/contains | london heathrow airport | london heathrow airport |\n",
      "| 39 | nan | what is the name of a neighborhood in <e> ? | 2 | Who was the father of the famous horse Cincinnati? | biology/organism/parents | location/place_with_neighborhoods/neighborhoods | cincinnati | cincinnati |\n",
      "| 40 | nan | what event took place in <e> in 1993 ? | 0 | what event tournament did uganda participate in | sports/sport_country/multi_event_tournaments_participated_in | location/location/events | uganda | uganda |\n",
      "| 41 | nan | what release can <e> be found on ? | 676 | What album can remember me be found on? | music/release_track/release | music/recording/releases | remember me | remember me |\n",
      "| 42 | nan | who was <e> | 114 | who boradcasts olivia newton-john | broadcast/artist/content | common/topic/notable_types | olivia newton-john | olivia newton - john |\n",
      "| 43 | nan | what has <e> appeared in | 134 | what famous entreprenour appeared in the daily show | tv/tv_program/episodes | common/topic/notable_types | the daily show | the daily show |\n",
      "| 44 | nan | where is the movie <e> from | 1374 | where is the movie music for madame from | film/film/country | film/film/country | music for madame | madame |\n",
      "| 45 | nan | where was <e> from | 2720 | where was nat king cole from | people/person/place_of_birth | music/release/region | nat \\\"king\\\" cole | nat king cole |\n",
      "| 46 | nan | what is the ninth track on the album <e> | 679 | travellin light was the fourth track on this album. | music/release/track_list | music/release/track | travellin' light | travellin light |\n",
      "| 47 | nan | what 's a drug with <e> in it | 23 | What's a hydrogen peroxide formula available in liquid | medicine/drug_dosage_form/formulations_available_in_this_form | medicine/drug_ingredient/active_ingredient_of_formulation | liquid | hydrogen peroxide |\n",
      "| 48 | nan | <e> works as what ? | 1 | does gerald j. rip work at the federal court | base/morelaw/canadian_judge/court | people/person/profession | gerald j. rip | gerald j. rip |\n",
      "| 49 | nan | what 's a song that <e> has written | 791 | What is a song that art of trance has written | music/artist/track | music/composer/compositions | art of trance | art of trance |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "import random\n",
    "\n",
    "predicted = []\n",
    "for index, row in tqdm_notebook(df.iterrows(), total=df.shape[0]):\n",
    "    if len(row['candidate_facts']) != 0:\n",
    "        candidate_relations = list(row['candidate_facts'].keys())\n",
    "        max_score = max(row['softmax_scores'])\n",
    "        predicted_relations = [r for i, r in enumerate(candidate_relations)\n",
    "                               if row['softmax_scores'][i] == max_score]\n",
    "        predicted_relation = random.choice(predicted_relations)\n",
    "        subject_mid, object_mids = random.choice(list(row['candidate_facts'][predicted_relation].items()))\n",
    "        predicted.append(tuple([predicted_relation, subject_mid, object_mids]))\n",
    "    else:\n",
    "        predicted.append(tuple([None, None, None]))\n",
    "\n",
    "evaluate(predicted)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Version: Softmax & Most Facts\n",
    "\n",
    "For the version 1 of softmax, we use the highest predicted relation and additionally guess the subject with the most objects associated with it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4422006a419946dba88b404831ea0fc2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7343f9d50e1e4818a81fbdcb1a51aa66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Subject & Relation Accuracy (SOTA 78.7%): 0.777286 [16857 of 21687]\n",
      "Object Accuracy: 0.829990 [18000 of 21687]\n",
      "Relation Accuracy (SOTA 88.4%): 0.875732 [18992 of 21687]\n",
      "Subject Accuracy (SOTA 79%): 0.830636 [18014 of 21687]\n",
      "Correct Interpretation: 0.912436 [19788 of 21687]\n",
      "No Interpretation: 0.019413 [421 of 21687]\n",
      "Negative Sample:\n",
      "| Index | Bucket | Closest False Predicate | True Relation Count | Question | True Relation | False Relation | Subject Name | Predicted Subject Name |\n",
      "| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n",
      "| 0 | nan | what town is in <e> | 39 | which town is in new york city | location/place_with_neighborhoods/neighborhoods | location/location/contains | new york city | new york city |\n",
      "| 1 | nan | what is a city in <e> , illinois | 1243 | what is a city in braxton county, west virginia | location/location/contains | location/us_county/hud_county_place | braxton county | braxton county |\n",
      "| 2 | nan | what is the license for <e> | 30 | whats the name of a law enforcement agency | organization/organization_sector/organizations_in_this_sector | computer/software/license | law enforcement agency | enforcement |\n",
      "| 3 | nan | what is a notable type of <e> ? | 1243 | What is an island located in united states minor outlying islands? | location/location/contains | common/topic/notable_types | united states minor outlying islands | outlying islands |\n",
      "| 4 | nan | in which country was <e> album released | 761 | in which country was the greatest hits album released | music/release/region | music/release/region | greatest hits | the greatest hits |\n",
      "| 5 | nan | what is an album recorded by <e> ? | 1044 | jackie is an album recorded by what artist? | music/album/artist | music/artist/album | jackie | jackie |\n",
      "| 6 | nan | what is <e> 's profession ? | 15 | What is cesky terrier's temperance? | biology/animal_breed/temperament | common/topic/notable_types | cesky terrier | cesky terrier |\n",
      "| 7 | nan | what is a live <e> by archer prewitt ? | 656 | what is the cover album by Japanese pop singer Ken Hirai | music/artist/album | music/album_release_type/albums | ken hirai | album |\n",
      "| 8 | nan | what is the area code for <e> | 0 | what is the parent institution for cohn-haddow center for judaic studies | base/jewlib/research_center/parent_institution | location/location/containedby | cohn-haddow center for judaic studies | cohn - haddow center for judaic studies |\n",
      "| 9 | nan | where is <e> from | 5 | where is produced by closer | theater/theater_production/performed_at | film/film/country | closer | closer |\n",
      "| 10 | nan | which state is <e> located in | 2472 | Which country is kenai fjords national park located in | location/location/containedby | base/usnationalparks/us_national_park/state | kenai fjords national park | kenai fjords national park |\n",
      "| 11 | nan | who was born in the <e> ? | 37 | What is a color of the st helens rlfc | sports/sports_team/colors | location/location/people_born_here | st helens rfc | st helens |\n",
      "| 12 | nan | which city is in <e> | 16 | which city is river dee, aberdeenshire in  | geography/river/cities | location/location/contains | river dee | dee , aberdeenshire |\n",
      "| 13 | nan | what country is <e> in | 1 | which city is in bosnian county | base/aareas/schema/administrative_area_type/subdivides_place | base/aareas/schema/administrative_area_type/pertains_to | bosnian county | bosnian county |\n",
      "| 14 | nan | what classification is <e> | 159 | what classification is the drug sodium sulfacetamide and sulfur 0.05 kit | medicine/drug_formulation/legal_status | medicine/drug_formulation/drug_category | sodium sulfacetamide and sulfur 0.05 kit | sodium sulfacetamide and sulfur 0.05 kit |\n",
      "| 15 | nan | what is <e> 's gender identity | 0 | what is mandy bright's sexual orientation  | base/adultentertainment/adult_entertainer/sexual_orientation | people/person/gender | mandy | mandy bright |\n",
      "| 16 | nan | which releases has <e> as a track ? | 646 | which releases have gold as the track? | music/recording/releases | music/release_track/release | gold | gold |\n",
      "| 17 | nan | what movie was directed by <e> | 1070 | what thriller was directed by Félix Enríquez Alcalá | film/film_genre/films_in_this_genre | film/director/film | thriller | felix enriquez alcala |\n",
      "| 18 | nan | what is the type of album that <e> is ? | 19 | what's the first episode of the fifth season of deja vu (the outer limits) | tv/tv_series_episode/next_episode | music/album/album_content_type | deja vu | the fifth season of |\n",
      "| 19 | nan | where did the film <e> originate ? | 441 | Where is the artist the lab originate  | music/artist/origin | film/film/country | the lab | the lab |\n",
      "| 20 | nan | what color is <e> | 7 | What color of horse is george washington | base/thoroughbredracing/thoroughbred_racehorse/color | education/educational_institution/colors | george washington | george washington |\n",
      "| 21 | nan | the song <e> is from which album | 676 | die in your arms is the single from which album  | music/release_track/release | music/release_track/recording | die in your arms | die in your arms |\n",
      "| 22 | nan | what is the title of a film with the <e> genre ? | 1070 | What is the drama film where black-and-white technique was used? | film/film_genre/films_in_this_genre | film/film_genre/films_in_this_genre | black-and-white | drama film |\n",
      "| 23 | nan | what entity governs the <e> | 123 | what American politician gave her name for the eleanor roosevelt national historic site | symbols/namesake/named_after | protected_sites/protected_site/governing_body | eleanor roosevelt national historic site | eleanor roosevelt national historic |\n",
      "| 24 | nan | what is <e> 's profession ? | 847 | What is aninuddha roy chowdhury's profession? | people/person/profession | common/topic/notable_types | aniruddha roy chowdhury | chowdhury |\n",
      "| 25 | nan | which genre does <e> fall under | 960 | What genre does casino royale fall under? | media_common/netflix_title/netflix_genres | book/book/genre | casino royale | casino royale |\n",
      "| 26 | nan | what fires were started as a result of <e> | 1 | what scandinavian airlines flight was involved in an accident | aviation/airline/accidents | base/fires/fire_cause/fires_caused_this_way | scandinavian airlines | accident |\n",
      "| 27 | nan | what mountains are located in the <e> | 1243 | what mountains are located within white mountains? | location/location/contains | geography/mountain_range/mountains | white mountains | white mountains |\n",
      "| 28 | nan | what genre is <e> video game ? | 169 | What is an adventure cvg game? | cvg/cvg_genre/games | cvg/computer_videogame/cvg_genre | adeventure | adventure |\n",
      "| 29 | nan | name a film that was shown at the <e> | 19 | What's a movie that was shown at the 2010 fantasia festival | film/film_festival_event/films | film/film_festival_event/films | 2010 fantasia film festival | 2010 fantasia festival |\n",
      "| 30 | nan | name an <e> person | 1070 | name a 2007 American film parody  | film/film_genre/films_in_this_genre | people/ethnicity/people | parody | american |\n",
      "| 31 | nan | which german city is <e> from | 1 | which department contains caroline bynum | education/academic/departments_old | people/person/place_of_birth | caroline bynum | caroline bynum |\n",
      "| 32 | nan | what 's one of the songs on the album <e> | 411 | What is one of the songs on the free album? | music/release_track/recording | music/release/track | free | free |\n",
      "| 33 | nan | is <e> pop music or folk | 1553 | is jane child pop music or j-pop | music/artist/genre | music/artist/genre | jane child | jane |\n",
      "| 34 | nan | who is the featured artist on the album <e> ? | 7 | what is the tittle of an entrepreneur who opens and runs restaurants professionally | people/profession/specializations | music/album/featured_artists | entrepreneur | opens |\n",
      "| 35 | nan | what is the country code for <e> | 1 | which country locates japanese city | base/aareas/schema/administrative_area_type/iso_country | base/aareas/schema/administrative_area_type/expected_codes | japanese city | japanese city |\n",
      "| 36 | nan | which biological process is <e> a part of | 0 | What kind of gas does a nitrogen fixation produce | base/process/process/output | base/change/process/part_of_process | nitrogen fixation | nitrogen fixation |\n",
      "| 37 | nan | what type of game is <e> | 226 | what type of game is the ncaa football 07 | cvg/computer_videogame/gameplay_modes | cvg/computer_videogame/cvg_genre | ncaa football 07 | ncaa football 07 |\n",
      "| 38 | nan | what college can be found on <e> | 18 | what college can be accessed by flying to the london heathrow airport | aviation/airport/serves | location/location/contains | london heathrow airport | london heathrow airport |\n",
      "| 39 | nan | what is the name of a neighborhood in <e> ? | 2 | Who was the father of the famous horse Cincinnati? | biology/organism/parents | location/place_with_neighborhoods/neighborhoods | cincinnati | cincinnati |\n",
      "| 40 | nan | what event took place in <e> in 1993 ? | 0 | what event tournament did uganda participate in | sports/sport_country/multi_event_tournaments_participated_in | location/location/events | uganda | uganda |\n",
      "| 41 | nan | what release can <e> be found on ? | 676 | What album can remember me be found on? | music/release_track/release | music/recording/releases | remember me | remember me |\n",
      "| 42 | nan | who was <e> | 114 | who boradcasts olivia newton-john | broadcast/artist/content | common/topic/notable_types | olivia newton-john | olivia newton - john |\n",
      "| 43 | nan | what has <e> appeared in | 134 | what famous entreprenour appeared in the daily show | tv/tv_program/episodes | common/topic/notable_types | the daily show | the daily show |\n",
      "| 44 | nan | where is the movie <e> from | 1374 | where is the movie music for madame from | film/film/country | film/film/country | music for madame | madame |\n",
      "| 45 | nan | where was <e> from | 2720 | where was nat king cole from | people/person/place_of_birth | music/release/region | nat \\\"king\\\" cole | nat king cole |\n",
      "| 46 | nan | what is the ninth track on the album <e> | 679 | travellin light was the fourth track on this album. | music/release/track_list | music/release/track | travellin' light | travellin light |\n",
      "| 47 | nan | what 's a drug with <e> in it | 23 | What's a hydrogen peroxide formula available in liquid | medicine/drug_dosage_form/formulations_available_in_this_form | medicine/drug_ingredient/active_ingredient_of_formulation | liquid | hydrogen peroxide |\n",
      "| 48 | nan | <e> works as what ? | 1 | does gerald j. rip work at the federal court | base/morelaw/canadian_judge/court | people/person/profession | gerald j. rip | gerald j. rip |\n",
      "| 49 | nan | what 's a song that <e> has written | 791 | What is a song that art of trance has written | music/artist/track | music/composer/compositions | art of trance | art of trance |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "import random\n",
    "\n",
    "predicted = []\n",
    "for index, row in tqdm_notebook(df.iterrows(), total=df.shape[0]):\n",
    "    if len(row['candidate_facts']) != 0:\n",
    "        candidate_relations = list(row['candidate_facts'].keys())\n",
    "        max_score = max(row['softmax_scores'])\n",
    "        predicted_relations = [r for i, r in enumerate(candidate_relations)\n",
    "                               if row['softmax_scores'][i] == max_score]\n",
    "        predicted_relation = random.choice(predicted_relations)\n",
    "        \n",
    "        # We use the `Better than random guessing` from notebook \n",
    "        # `HYPOTHESIS - Question Refers to Multiple Subjects`.\n",
    "        subject_mid, object_mids = sorted(row['candidate_facts'][predicted_relation].items(),\n",
    "                                          key=lambda i: len(i[1]), reverse=True)[0]\n",
    "        predicted.append(tuple([predicted_relation, subject_mid, object_mids]))\n",
    "    else:\n",
    "        predicted.append(tuple([None, None, None]))\n",
    "\n",
    "evaluate(predicted)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Version: Softmax & Most Facts & Entity Relation Distribution\n",
    "\n",
    "The hypothesis of version 2 is that we can use the distribution of relations for a particular set of facts to our advantage. Particular entities favor some relations more than others; therefore, we use that probability in a product of experts model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0779fdfe1593475d88e5d72dcfc828b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79224b854b184f9ab5af52e6c15f0bc7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Subject & Relation Accuracy (SOTA 78.7%): 0.781067 [16939 of 21687]\n",
      "Object Accuracy: 0.835662 [18123 of 21687]\n",
      "Relation Accuracy (SOTA 88.4%): 0.881680 [19121 of 21687]\n",
      "Subject Accuracy (SOTA 79%): 0.830451 [18010 of 21687]\n",
      "Correct Interpretation: 0.915479 [19854 of 21687]\n",
      "No Interpretation: 0.019413 [421 of 21687]\n",
      "Negative Sample:\n",
      "| Index | Bucket | Closest False Predicate | True Relation Count | Question | True Relation | False Relation | Subject Name | Predicted Subject Name |\n",
      "| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n",
      "| 0 | nan | ['what town is in <e>', 'which city is in <e>', 'what town is in <e> ?'] | 39 | which town is in new york city | location/place_with_neighborhoods/neighborhoods | location/location/contains | new york city | new york city |\n",
      "| 1 | nan | ['what types of hotel is <e>', 'what type of location is the <e>', 'what accommodation type is <e> ?'] | 1 | who operates the fairfield inn & suites austin northwest/arboretum | travel/hotel/operated_by | travel/accommodation/accommodation_type | fairfield inn & suites austin northwest/arboretum | fairfield inn & suites austin northwest / arboretum |\n",
      "| 2 | nan | ['what is the license for <e>', 'what license does <e> have', 'what license does <e> have'] | 30 | whats the name of a law enforcement agency | organization/organization_sector/organizations_in_this_sector | computer/software/license | law enforcement agency | enforcement |\n",
      "| 3 | nan | ['what is a notable type of <e> ?', 'what is a notable type of a <e> ?', 'what is a notable type about <e> ?'] | 1243 | What is an island located in united states minor outlying islands? | location/location/contains | common/topic/notable_types | united states minor outlying islands | outlying islands |\n",
      "| 4 | nan | ['in which country was <e> album released', 'in which country was <e> album released', 'in what country was <e> album released'] | 761 | in which country was the greatest hits album released | music/release/region | music/release/region | greatest hits | the greatest hits |\n",
      "| 5 | nan | [\"what 's an example of a <e> organization\"] | 2 | What's an ad campaign focused on health care | user/micahsaul/advertising/advertised_thing/ad_campaigns | government/government_service/operating_organization | health care | health care |\n",
      "| 6 | nan | ['what is an album recorded by <e> ?', 'what is an album recorded by <e> ?', 'what is an album recorded by <e> ?'] | 1044 | jackie is an album recorded by what artist? | music/album/artist | music/artist/album | jackie | jackie |\n",
      "| 7 | nan | ['what is a live <e> by archer prewitt ?', 'what is an <e> released by a dominican singer in 1998', 'what is an <e> by jetplane landing ?'] | 656 | what is the cover album by Japanese pop singer Ken Hirai | music/artist/album | music/album_release_type/albums | ken hirai | album |\n",
      "| 8 | nan | ['what is the area code for <e>', 'what is the location of <e>', 'what is the location of <e>'] | 0 | what is the parent institution for cohn-haddow center for judaic studies | base/jewlib/research_center/parent_institution | location/location/containedby | cohn-haddow center for judaic studies | cohn - haddow center for judaic studies |\n",
      "| 9 | nan | ['where released <e>', 'where is <e>', 'where is <e> from'] | 5 | where is produced by closer | theater/theater_production/performed_at | music/release/region | closer | closer |\n",
      "| 10 | nan | ['which state is <e> located in', 'which state is <e> located in ?', 'which state is <e> in'] | 2472 | Which country is kenai fjords national park located in | location/location/containedby | base/usnationalparks/us_national_park/state | kenai fjords national park | kenai fjords national park |\n",
      "| 11 | nan | ['who was born in the <e> ?', 'who is an author from <e> ?', 'who was born in the <e> ?'] | 37 | What is a color of the st helens rlfc | sports/sports_team/colors | location/location/people_born_here | st helens rfc | st helens |\n",
      "| 12 | nan | ['which city is in <e>', 'what city is in <e> ?', 'what city is in <e> ?'] | 16 | which city is river dee, aberdeenshire in  | geography/river/cities | location/location/contains | river dee | dee , aberdeenshire |\n",
      "| 13 | nan | ['what country is <e> in', 'what does <e> pertain to', 'what does the <e> belong to'] | 1 | which city is in bosnian county | base/aareas/schema/administrative_area_type/subdivides_place | base/aareas/schema/administrative_area_type/pertains_to | bosnian county | bosnian county |\n",
      "| 14 | nan | ['what classification is <e>', 'what kind of classification is <e> given', 'what is the classification of <e>'] | 159 | what classification is the drug sodium sulfacetamide and sulfur 0.05 kit | medicine/drug_formulation/legal_status | medicine/drug_formulation/drug_category | sodium sulfacetamide and sulfur 0.05 kit | sodium sulfacetamide and sulfur 0.05 kit |\n",
      "| 15 | nan | [\"what is <e> 's gender identity\", \"what is <e> 's gender identity\", \"what is <e> 's sex ?\"] | 0 | what is mandy bright's sexual orientation  | base/adultentertainment/adult_entertainer/sexual_orientation | people/person/gender | mandy | mandy bright |\n",
      "| 16 | nan | ['which releases has <e> as a track ?', 'which release has the track <e>', 'which releases has the track <e> ?'] | 646 | which releases have gold as the track? | music/recording/releases | music/release_track/release | gold | gold |\n",
      "| 17 | nan | ['what movie was directed by <e>', 'what movie was directed by <e>', 'what film was directed by <e>'] | 1070 | what thriller was directed by Félix Enríquez Alcalá | film/film_genre/films_in_this_genre | film/director/film | thriller | felix enriquez alcala |\n",
      "| 18 | nan | ['what is the country of origin of the movie <e> ?', 'what is the country of origin of the film <e> ?', 'what is the country of origin for the film <e> ?'] | 19 | what's the first episode of the fifth season of deja vu (the outer limits) | tv/tv_series_episode/next_episode | film/film/country | deja vu | the fifth season of |\n",
      "| 19 | nan | ['where did the film <e> originate ?', 'where did the film <e> originate ?', 'where did the film <e> originate ?'] | 441 | Where is the artist the lab originate  | music/artist/origin | film/film/country | the lab | the lab |\n",
      "| 20 | nan | ['what type of storage does the <e> support', 'what type of storage card does the <e> use', 'what types of card does <e> need'] | 2 | What type of sensor does the nikon d3 have? | digicams/digital_camera/sensor_type | digicams/digital_camera/supported_storage_types | nikon d3 | nikon d3 |\n",
      "| 21 | nan | ['what color is <e>', 'what color is worn by <e> ?', 'what color is <e>'] | 7 | What color of horse is george washington | base/thoroughbredracing/thoroughbred_racehorse/color | education/educational_institution/colors | george washington | george washington |\n",
      "| 22 | nan | ['the song <e> is from which album', '<e> is from what album', 'what is a song from the album <e>'] | 676 | die in your arms is the single from which album  | music/release_track/release | music/release_track/recording | die in your arms | die in your arms |\n",
      "| 23 | nan | ['what is the title of a film with the <e> genre ?', 'what is the name of a film where the genre is <e>', 'what is the name of an italian <e> ?'] | 1070 | What is the drama film where black-and-white technique was used? | film/film_genre/films_in_this_genre | film/film_genre/films_in_this_genre | black-and-white | drama film |\n",
      "| 24 | nan | ['what entity governs the <e>', 'what is the governing body over the <e>', 'who is in charge of the <e>'] | 123 | what American politician gave her name for the eleanor roosevelt national historic site | symbols/namesake/named_after | protected_sites/protected_site/governing_body | eleanor roosevelt national historic site | eleanor roosevelt national historic |\n",
      "| 25 | nan | [\"what is <e> 's profession ?\", \"what is <e> 's profession ?\", \"what is <e> 's profession ?\"] | 847 | What is aninuddha roy chowdhury's profession? | people/person/profession | common/topic/notable_types | aniruddha roy chowdhury | chowdhury |\n",
      "| 26 | nan | ['what time of media is <e>', 'what type of music is <e>', 'what type of model is <e>'] | 6 | What type of media is lemon | type/content/media_type | common/topic/notable_types | lemon | lemon |\n",
      "| 27 | nan | ['what fires were started as a result of <e>'] | 1 | what scandinavian airlines flight was involved in an accident | aviation/airline/accidents | base/fires/fire_cause/fires_caused_this_way | scandinavian airlines | accident |\n",
      "| 28 | nan | ['what mountains are located in the <e>', 'what mountain is located in the <e>', 'what mountain is in the <e> ?'] | 1243 | what mountains are located within white mountains? | location/location/contains | geography/mountain_range/mountains | white mountains | white mountains |\n",
      "| 29 | nan | ['what is a <e> group', 'what is a <e> band', \"what 's a <e> band\"] | 727 | what is an electro-industrial genre  | music/genre/albums | music/genre/artists | electro-industrial | electro - industrial |\n",
      "| 30 | nan | ['what is a mode of <e>', 'what is a mode in <e>', 'what is a feature of <e>'] | 169 | What is an adventure cvg game? | cvg/cvg_genre/games | cvg/computer_videogame/gameplay_modes | adeventure | adventure |\n",
      "| 31 | nan | ['name a film that was shown at the <e>', 'what film was shown at the <e>', 'what film was shown at the <e>'] | 19 | What's a movie that was shown at the 2010 fantasia festival | film/film_festival_event/films | film/film_festival_event/films | 2010 fantasia film festival | 2010 fantasia festival |\n",
      "| 32 | nan | ['name an <e> person', 'name a <e> .', 'who is a <e> mayor'] | 1070 | name a 2007 American film parody  | film/film_genre/films_in_this_genre | people/ethnicity/people | parody | american |\n",
      "| 33 | nan | ['which german city is <e> from', 'which european city is <e> from', 'which state is <e> from'] | 1 | which department contains caroline bynum | education/academic/departments_old | people/person/place_of_birth | caroline bynum | caroline bynum |\n",
      "| 34 | nan | [\"what 's one of the songs on the album <e>\", \"what 's one of the songs on the album <e>\", \"what 's one of the songs on the release <e>\"] | 411 | What is one of the songs on the free album? | music/release_track/recording | music/release/track | free | free |\n",
      "| 35 | nan | ['is <e> pop music or folk', 'is <e> rock music or hip - hop', 'is <e> dance - pop or k - pop'] | 1553 | is jane child pop music or j-pop | music/artist/genre | music/artist/genre | jane child | jane |\n",
      "| 36 | nan | ['who is the featured artist on the album <e> ?', 'what pretty girl rock singer is featured on <e>', 'who is a featured artist on the album <e> ?'] | 7 | what is the tittle of an entrepreneur who opens and runs restaurants professionally | people/profession/specializations | music/album/featured_artists | entrepreneur | opens |\n",
      "| 37 | nan | ['what is the country code for <e>', 'what is the country code for <e>', 'what are the expected codes of <e> ?'] | 1 | which country locates japanese city | base/aareas/schema/administrative_area_type/iso_country | base/aareas/schema/administrative_area_type/expected_codes | japanese city | japanese city |\n",
      "| 38 | nan | ['which biological process is <e> a part of'] | 0 | What kind of gas does a nitrogen fixation produce | base/process/process/output | base/change/process/part_of_process | nitrogen fixation | nitrogen fixation |\n",
      "| 39 | nan | ['what type of game is <e>', 'what type of game is <e>', 'what type of game is <e>'] | 226 | what type of game is the ncaa football 07 | cvg/computer_videogame/gameplay_modes | cvg/computer_videogame/cvg_genre | ncaa football 07 | ncaa football 07 |\n",
      "| 40 | nan | ['what is the name of a woman born in <e> ?', 'what is the name of a man born in <e> ?', 'who was the scientist born in <e> ?'] | 2 | Who was the father of the famous horse Cincinnati? | biology/organism/parents | location/location/people_born_here | cincinnati | cincinnati |\n",
      "| 41 | nan | ['what event took place in <e> in 1993 ?', 'what event took place at <e> in 1943 ?', 'what event took place in <e> ?'] | 0 | what event tournament did uganda participate in | sports/sport_country/multi_event_tournaments_participated_in | location/location/events | uganda | uganda |\n",
      "| 42 | nan | ['what is the genre of <e> .', 'what is the genre of <e> ?', 'what is the genre of <e> ?'] | 1553 | what is the genre of trout fishing in america's work | music/artist/genre | book/book/genre | trout fishing in america | trout fishing in america |\n",
      "| 43 | nan | ['who is a parent of <e>', 'who is father of <e>', 'who is a parent of <e>'] | 114 | who boradcasts olivia newton-john | broadcast/artist/content | people/person/parents | olivia newton-john | olivia newton - john |\n",
      "| 44 | nan | ['where is the movie <e> from', 'where is the movie <e> from ?', 'where is the film <e> from'] | 1374 | where is the movie music for madame from | film/film/country | film/film/country | music for madame | madame |\n",
      "| 45 | nan | ['where was <e> from', 'where is <e> from', 'where is <e> from'] | 2720 | where was nat king cole from | people/person/place_of_birth | music/release/region | nat \\\"king\\\" cole | nat king cole |\n",
      "| 46 | nan | ['what is the ninth track on the album <e>', 'what was a track on the album <e>', 'what was a track on the album <e>'] | 679 | travellin light was the fourth track on this album. | music/release/track_list | music/release/track | travellin' light | travellin light |\n",
      "| 47 | nan | [\"what 's a drug with <e> in it\", \"what 's a drug that uses <e>\", 'what drug has <e> as the active ingredient'] | 23 | What's a hydrogen peroxide formula available in liquid | medicine/drug_dosage_form/formulations_available_in_this_form | medicine/drug_ingredient/active_ingredient_of_formulation | liquid | hydrogen peroxide |\n",
      "| 48 | nan | ['does <e> follow taoism or buddhism', \"what is <e> 's theology\", '<e> ascribes to what church ?'] | 1 | does gerald j. rip work at the federal court | base/morelaw/canadian_judge/court | people/person/religion | gerald j. rip | gerald j. rip |\n",
      "| 49 | nan | ['<e> is what gender ?', '<e> is what gender ?', '<e> self - identifies as which gender ?'] | 1928 | john binder identifies with what gender? | people/person/gender | people/person/gender | john binden | john binder |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "import random\n",
    "\n",
    "predicted = []\n",
    "for index, row in tqdm_notebook(df.iterrows(), total=df.shape[0]):\n",
    "    if len(row['candidate_facts']) != 0:\n",
    "        candidate_relations = list(row['candidate_facts'].keys())\n",
    "        kg_relation_probability = [sum(len(row['candidate_facts'][r][mid]) for mid in row['candidate_facts'][r])\n",
    "                                     for r in candidate_relations]\n",
    "        scores = [(row['softmax_scores'][i] *\n",
    "                   transform_probability_from_kg_to_train[r] *\n",
    "                   kg_relation_probability[i]) for i, r in enumerate(candidate_relations)]\n",
    "        max_score = max(scores)\n",
    "        predicted_relations = [r for i, r in enumerate(candidate_relations) if scores[i] == max_score]\n",
    "        predicted_relation = random.choice(predicted_relations)\n",
    "        \n",
    "        # We use the `Better than random guessing` from notebook \n",
    "        # `HYPOTHESIS - Question Refers to Multiple Subjects`.\n",
    "        subject_mid, object_mids = sorted(row['candidate_facts'][predicted_relation].items(),\n",
    "                                          key=lambda i: len(i[1]), reverse=True)[0]\n",
    "        predicted.append(tuple([predicted_relation, subject_mid, object_mids]))\n",
    "    else:\n",
    "        predicted.append(tuple([None, None, None]))\n",
    "\n",
    "evaluate(predicted)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Numbers:**\n",
    "\n",
    "- Subject & Relation Accuracy (SOTA 78.7%): 0.781067 [16939 of 21687]\n",
    "- Object Accuracy: 0.835662 [18123 of 21687]\n",
    "- Relation Accuracy (SOTA 88.4%): 0.881680 [19121 of 21687]\n",
    "- Subject Accuracy (SOTA 79%): 0.830451 [18010 of 21687]\n",
    "- Correct Interpretation: 0.915479 [19854 of 21687]\n",
    "- No Interpretation: 0.019413 [421 of 21687]\n",
    "\n",
    "**Discussion:**\n",
    "\n",
    "Following the error bucket given that 14/50 were marked correctly, we revaluate our model at 0.065108 * (14/ 50) + 0.915479 = 0.93370924% correct.\n",
    "\n",
    "Given that 34/50 are incorrect, excluding noise, we find that there exists at most 0.065108 * (34/50) = 4.4\\% room for improvement with 0.02083456 (2.1%) due to modeling low shot relations.\n",
    "\n",
    "**Error Buckets: (1 - (0.019413 + 0.915479) = 0.065108% of Examples)**\n",
    "\n",
    "**Buckets:**\n",
    "- Incorrect, Low (16/50): The relation choosen was incorrect and there are less than 7 examples in the training data for the true relation.\n",
    "- Correct (14/50): The model interpreted the question correctly. \n",
    "- Incorrect (36/50): The model interpreted the question incorrectly.\n",
    "- Incorrect, Noise (2/50): The question was interpreted incorrectly and did not make sense.\n",
    "- Incorrect, Relation (30/50): The model failed to get the correct relation.\n",
    "- Incorrect, Subject (14/50): The model failed to get the correct subject.\n",
    "\n",
    "\n",
    "| Index | Bucket | Closest False Predicate | True Relation Count | Question | True Relation | False Relation | Subject Name | Predicted Subject Name |\n",
    "| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n",
    "| 0 | Correct | ['what town is in <e>', 'which city is in <e>', 'what town is in <e> ?'] | 39 | which town is in new york city | location/place_with_neighborhoods/neighborhoods | location/location/contains | new york city | new york city |\n",
    "| 1 | Incorrect, Low, Relation | ['what types of hotel is <e>', 'what type of location is the <e>', 'what accommodation type is <e> ?'] | 1 | who operates the fairfield inn & suites austin northwest/arboretum | travel/hotel/operated_by | travel/accommodation/accommodation_type | fairfield inn & suites austin northwest/arboretum | fairfield inn & suites austin northwest / arboretum |\n",
    "| 2 | Incorrect, Subject, Relation | ['what is the license for <e>', 'what license does <e> have', 'what license does <e> have'] | 30 | whats the name of a law enforcement agency | organization/organization_sector/organizations_in_this_sector | computer/software/license | law enforcement agency | enforcement |\n",
    "| 3 | Incorrect, Subject, Relation | ['what is a notable type of <e> ?', 'what is a notable type of a <e> ?', 'what is a notable type about <e> ?'] | 1243 | What is an island located in united states minor outlying islands? | location/location/contains | common/topic/notable_types | united states minor outlying islands | outlying islands |\n",
    "| 4 | Correct | ['in which country was <e> album released', 'in which country was <e> album released', 'in what country was <e> album released'] | 761 | in which country was the greatest hits album released | music/release/region | music/release/region | greatest hits | the greatest hits |\n",
    "| 5 | Incorrect, Low, Relation | [\"what 's an example of a <e> organization\"] | 2 | What's an ad campaign focused on health care | user/micahsaul/advertising/advertised_thing/ad_campaigns | government/government_service/operating_organization | health care | health care |\n",
    "| 6 | Incorrect | ['what is an album recorded by <e> ?', 'what is an album recorded by <e> ?', 'what is an album recorded by <e> ?'] | 1044 | jackie is an album recorded by what artist? | music/album/artist | music/artist/album | jackie | jackie |\n",
    "| 7 | Correct | ['what is a live <e> by archer prewitt ?', 'what is an <e> released by a dominican singer in 1998', 'what is an <e> by jetplane landing ?'] | 656 | what is the cover album by Japanese pop singer Ken Hirai | music/artist/album | music/album_release_type/albums | ken hirai | album |\n",
    "| 8 | Incorrect, Low, Relation | ['what is the area code for <e>', 'what is the location of <e>', 'what is the location of <e>'] | 0 | what is the parent institution for cohn-haddow center for judaic studies | base/jewlib/research_center/parent_institution | location/location/containedby | cohn-haddow center for judaic studies | cohn - haddow center for judaic studies |\n",
    "| 9 | Incorrect, Low, Relation | ['where released <e>', 'where is <e>', 'where is <e> from'] | 5 | where is produced by closer | theater/theater_production/performed_at | music/release/region | closer | closer |\n",
    "| 10 | Incorrect, Relation | ['which state is <e> located in', 'which state is <e> located in ?', 'which state is <e> in'] | 2472 | Which country is kenai fjords national park located in | location/location/containedby | base/usnationalparks/us_national_park/state | kenai fjords national park | kenai fjords national park |\n",
    "| 11 | Incorrect, Relation | ['who was born in the <e> ?', 'who is an author from <e> ?', 'who was born in the <e> ?'] | 37 | What is a color of the st helens rlfc | sports/sports_team/colors | location/location/people_born_here | st helens rfc | st helens |\n",
    "| 12 | Incorrect, Subject, Relation | ['which city is in <e>', 'what city is in <e> ?', 'what city is in <e> ?'] | 16 | which city is river dee, aberdeenshire in  | geography/river/cities | location/location/contains | river dee | dee , aberdeenshire |\n",
    "| 13 | Incorrect, Low, Relation | ['what country is <e> in', 'what does <e> pertain to', 'what does the <e> belong to'] | 1 | which city is in bosnian county | base/aareas/schema/administrative_area_type/subdivides_place | base/aareas/schema/administrative_area_type/pertains_to | bosnian county | bosnian county |\n",
    "| 14 | Correct | ['what classification is <e>', 'what kind of classification is <e> given', 'what is the classification of <e>'] | 159 | what classification is the drug sodium sulfacetamide and sulfur 0.05 kit | medicine/drug_formulation/legal_status | medicine/drug_formulation/drug_category | sodium sulfacetamide and sulfur 0.05 kit | sodium sulfacetamide and sulfur 0.05 kit |\n",
    "| 15 | Incorrect, Low, Subject, Relation | [\"what is <e> 's gender identity\", \"what is <e> 's gender identity\", \"what is <e> 's sex ?\"] | 0 | what is mandy bright's sexual orientation  | base/adultentertainment/adult_entertainer/sexual_orientation | people/person/gender | mandy | mandy bright |\n",
    "| 16 | Correct | ['which releases has <e> as a track ?', 'which release has the track <e>', 'which releases has the track <e> ?'] | 646 | which releases have gold as the track? | music/recording/releases | music/release_track/release | gold | gold |\n",
    "| 17 | Incorrect, Subject, Relation | ['what movie was directed by <e>', 'what movie was directed by <e>', 'what film was directed by <e>'] | 1070 | what thriller was directed by Félix Enríquez Alcalá | film/film_genre/films_in_this_genre | film/director/film | thriller | felix enriquez alcala |\n",
    "| 18 | Incorrect, Subject, Relation | ['what is the country of origin of the movie <e> ?', 'what is the country of origin of the film <e> ?', 'what is the country of origin for the film <e> ?'] | 19 | what's the first episode of the fifth season of deja vu (the outer limits) | tv/tv_series_episode/next_episode | film/film/country | deja vu | the fifth season of |\n",
    "| 19 | Incorrect, Relation | ['where did the film <e> originate ?', 'where did the film <e> originate ?', 'where did the film <e> originate ?'] | 441 | Where is the artist the lab originate  | music/artist/origin | film/film/country | the lab | the lab |\n",
    "| 20 | Incorrect, Low, Relation | ['what type of storage does the <e> support', 'what type of storage card does the <e> use', 'what types of card does <e> need'] | 2 | What type of sensor does the nikon d3 have? | digicams/digital_camera/sensor_type | digicams/digital_camera/supported_storage_types | nikon d3 | nikon d3 |\n",
    "| 21 | Incorrect, Low, Relation | ['what color is <e>', 'what color is worn by <e> ?', 'what color is <e>'] | 7 | What color of horse is george washington | base/thoroughbredracing/thoroughbred_racehorse/color | education/educational_institution/colors | george washington | george washington |\n",
    "| 22 | Correct | ['the song <e> is from which album', '<e> is from what album', 'what is a song from the album <e>'] | 676 | die in your arms is the single from which album  | music/release_track/release | music/release_track/recording | die in your arms | die in your arms |\n",
    "| 23 | Incorrect, Subject, Relation | ['what is the title of a film with the <e> genre ?', 'what is the name of a film where the genre is <e>', 'what is the name of an italian <e> ?'] | 1070 | What is the drama film where black-and-white technique was used? | film/film_genre/films_in_this_genre | film/film_genre/films_in_this_genre | black-and-white | drama film |\n",
    "| 24 | Incorrect | ['what entity governs the <e>', 'what is the governing body over the <e>', 'who is in charge of the <e>'] | 123 | what American politician gave her name for the eleanor roosevelt national historic site | symbols/namesake/named_after | protected_sites/protected_site/governing_body | eleanor roosevelt national historic site | eleanor roosevelt national historic |\n",
    "| 25 | Incorrect, Subject | [\"what is <e> 's profession ?\", \"what is <e> 's profession ?\", \"what is <e> 's profession ?\"] | 847 | What is aninuddha roy chowdhury's profession? | people/person/profession | common/topic/notable_types | aniruddha roy chowdhury | chowdhury |\n",
    "| 26 | Correct | ['what time of media is <e>', 'what type of music is <e>', 'what type of model is <e>'] | 6 | What type of media is lemon | type/content/media_type | common/topic/notable_types | lemon | lemon |\n",
    "| 27 | Incorrect, Low, Subject, Relation | ['what fires were started as a result of <e>'] | 1 | what scandinavian airlines flight was involved in an accident | aviation/airline/accidents | base/fires/fire_cause/fires_caused_this_way | scandinavian airlines | accident |\n",
    "| 28 | Correct | ['what mountains are located in the <e>', 'what mountain is located in the <e>', 'what mountain is in the <e> ?'] | 1243 | what mountains are located within white mountains? | location/location/contains | geography/mountain_range/mountains | white mountains | white mountains |\n",
    "| 29 | Incorrect, Noise, Relation | ['what is a <e> group', 'what is a <e> band', \"what 's a <e> band\"] | 727 | what is an electro-industrial genre  | music/genre/albums | music/genre/artists | electro-industrial | electro - industrial |\n",
    "| 30 | Incorrect, Relation | ['what is a mode of <e>', 'what is a mode in <e>', 'what is a feature of <e>'] | 169 | What is an adventure cvg game? | cvg/cvg_genre/games | cvg/computer_videogame/gameplay_modes | adeventure | adventure |\n",
    "| 31 | Correct | ['name a film that was shown at the <e>', 'what film was shown at the <e>', 'what film was shown at the <e>'] | 19 | What's a movie that was shown at the 2010 fantasia festival | film/film_festival_event/films | film/film_festival_event/films | 2010 fantasia film festival | 2010 fantasia festival |\n",
    "| 32 | Incorrect, Subject, Relation | ['name an <e> person', 'name a <e> .', 'who is a <e> mayor'] | 1070 | name a 2007 American film parody  | film/film_genre/films_in_this_genre | people/ethnicity/people | parody | american |\n",
    "| 33 | Incorrect, Low, Relation | ['which german city is <e> from', 'which european city is <e> from', 'which state is <e> from'] | 1 | which department contains caroline bynum | education/academic/departments_old | people/person/place_of_birth | caroline bynum | caroline bynum |\n",
    "| 34 | Correct | [\"what 's one of the songs on the album <e>\", \"what 's one of the songs on the album <e>\", \"what 's one of the songs on the release <e>\"] | 411 | What is one of the songs on the free album? | music/release_track/recording | music/release/track | free | free |\n",
    "| 35 | Incorrect, Subject | ['is <e> pop music or folk', 'is <e> rock music or hip - hop', 'is <e> dance - pop or k - pop'] | 1553 | is jane child pop music or j-pop | music/artist/genre | music/artist/genre | jane child | jane |\n",
    "| 36 | Incorrect, Low, Subject, Relation | ['who is the featured artist on the album <e> ?', 'what pretty girl rock singer is featured on <e>', 'who is a featured artist on the album <e> ?'] | 7 | what is the tittle of an entrepreneur who opens and runs restaurants professionally | people/profession/specializations | music/album/featured_artists | entrepreneur | opens |\n",
    "| 37 | Incorrect, Low, Relation | ['what is the country code for <e>', 'what is the country code for <e>', 'what are the expected codes of <e> ?'] | 1 | which country locates japanese city | base/aareas/schema/administrative_area_type/iso_country | base/aareas/schema/administrative_area_type/expected_codes | japanese city | japanese city |\n",
    "| 38 | Incorrect, Low, Relation | ['which biological process is <e> a part of'] | 0 | What kind of gas does a nitrogen fixation produce | base/process/process/output | base/change/process/part_of_process | nitrogen fixation | nitrogen fixation |\n",
    "| 39 | Correct | ['what type of game is <e>', 'what type of game is <e>', 'what type of game is <e>'] | 226 | what type of game is the ncaa football 07 | cvg/computer_videogame/gameplay_modes | cvg/computer_videogame/cvg_genre | ncaa football 07 | ncaa football 07 |\n",
    "| 40 | Incorrect, Low, Relation | ['what is the name of a woman born in <e> ?', 'what is the name of a man born in <e> ?', 'who was the scientist born in <e> ?'] | 2 | Who was the father of the famous horse Cincinnati? | biology/organism/parents | location/location/people_born_here | cincinnati | cincinnati |\n",
    "| 41 | Incorrect, Low, Relation | ['what event took place in <e> in 1993 ?', 'what event took place at <e> in 1943 ?', 'what event took place in <e> ?'] | 0 | what event tournament did uganda participate in | sports/sport_country/multi_event_tournaments_participated_in | location/location/events | uganda | uganda |\n",
    "| 42 | Correct | ['what is the genre of <e> .', 'what is the genre of <e> ?', 'what is the genre of <e> ?'] | 1553 | what is the genre of trout fishing in america's work | music/artist/genre | book/book/genre | trout fishing in america | trout fishing in america |\n",
    "| 43 | Incorrect, Relation | ['who is a parent of <e>', 'who is father of <e>', 'who is a parent of <e>'] | 114 | who boradcasts olivia newton-john | broadcast/artist/content | people/person/parents | olivia newton-john | olivia newton - john |\n",
    "| 44 | Incorrect, Subject | ['where is the movie <e> from', 'where is the movie <e> from ?', 'where is the film <e> from'] | 1374 | where is the movie music for madame from | film/film/country | film/film/country | music for madame | madame |\n",
    "| 45 | Correct | ['where was <e> from', 'where is <e> from', 'where is <e> from'] | 2720 | where was nat king cole from | people/person/place_of_birth | music/release/region | nat \\\"king\\\" cole | nat king cole |\n",
    "| 46 | Incorrect, Noise | ['what is the ninth track on the album <e>', 'what was a track on the album <e>', 'what was a track on the album <e>'] | 679 | travellin light was the fourth track on this album. | music/release/track_list | music/release/track | travellin' light | travellin light |\n",
    "| 47 | Incorrect, Relation, Subject | [\"what 's a drug with <e> in it\", \"what 's a drug that uses <e>\", 'what drug has <e> as the active ingredient'] | 23 | What's a hydrogen peroxide formula available in liquid | medicine/drug_dosage_form/formulations_available_in_this_form | medicine/drug_ingredient/active_ingredient_of_formulation | liquid | hydrogen peroxide |\n",
    "| 48 | Incorrect, Low, Relation | ['does <e> follow taoism or buddhism', \"what is <e> 's theology\", '<e> ascribes to what church ?'] | 1 | does gerald j. rip work at the federal court | base/morelaw/canadian_judge/court | people/person/religion | gerald j. rip | gerald j. rip |\n",
    "| 49 | Correct | ['<e> is what gender ?', '<e> is what gender ?', '<e> self - identifies as which gender ?'] | 1928 | john binder identifies with what gender? | people/person/gender | people/person/gender | john binden | john binder |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Version: Softmax & Most Facts & Entity Relation Distribution & Ensemble\n",
    "\n",
    "In version 3, we try to ensemble 3 similary performing softmax models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bd4ed37335e409ab2f9948831cc0f03",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8ba31b743544b92afdfcf84ba5e5c0a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21687), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Subject & Relation Accuracy (SOTA 78.7%): 0.780929 [16936 of 21687]\n",
      "Object Accuracy: 0.835846 [18127 of 21687]\n",
      "Relation Accuracy (SOTA 88.4%): 0.882003 [19128 of 21687]\n",
      "Subject Accuracy (SOTA 79%): 0.830405 [18009 of 21687]\n",
      "Correct Interpretation: 0.915433 [19853 of 21687]\n",
      "No Interpretation: 0.019413 [421 of 21687]\n",
      "Negative Sample:\n",
      "| Index | Bucket | Closest False Predicate | True Relation Count | Question | True Relation | False Relation | Subject Name | Predicted Subject Name |\n",
      "| --- | --- | --- | --- | --- | --- | --- | --- | --- |\n",
      "| 0 | nan | what town is in <e> | 39 | which town is in new york city | location/place_with_neighborhoods/neighborhoods | location/location/contains | new york city | new york city |\n",
      "| 1 | nan | what is the license for <e> | 30 | whats the name of a law enforcement agency | organization/organization_sector/organizations_in_this_sector | computer/software/license | law enforcement agency | enforcement |\n",
      "| 2 | nan | what is a notable type of <e> ? | 1243 | What is an island located in united states minor outlying islands? | location/location/contains | common/topic/notable_types | united states minor outlying islands | outlying islands |\n",
      "| 3 | nan | in which country was <e> album released | 761 | in which country was the greatest hits album released | music/release/region | music/release/region | greatest hits | the greatest hits |\n",
      "| 4 | nan | what 's an example of a <e> organization | 2 | What's an ad campaign focused on health care | user/micahsaul/advertising/advertised_thing/ad_campaigns | government/government_service/operating_organization | health care | health care |\n",
      "| 5 | nan | what is an album recorded by <e> ? | 1044 | jackie is an album recorded by what artist? | music/album/artist | music/artist/album | jackie | jackie |\n",
      "| 6 | nan | what is a live <e> by archer prewitt ? | 656 | what is the cover album by Japanese pop singer Ken Hirai | music/artist/album | music/album_release_type/albums | ken hirai | album |\n",
      "| 7 | nan | what is the area code for <e> | 0 | what is the parent institution for cohn-haddow center for judaic studies | base/jewlib/research_center/parent_institution | location/location/containedby | cohn-haddow center for judaic studies | cohn - haddow center for judaic studies |\n",
      "| 8 | nan | where released <e> | 5 | where is produced by closer | theater/theater_production/performed_at | music/release/region | closer | closer |\n",
      "| 9 | nan | which state is <e> located in | 2472 | Which country is kenai fjords national park located in | location/location/containedby | base/usnationalparks/us_national_park/state | kenai fjords national park | kenai fjords national park |\n",
      "| 10 | nan | who was born in the <e> ? | 37 | What is a color of the st helens rlfc | sports/sports_team/colors | location/location/people_born_here | st helens rfc | st helens |\n",
      "| 11 | nan | which city is in <e> | 16 | which city is river dee, aberdeenshire in  | geography/river/cities | location/location/contains | river dee | dee , aberdeenshire |\n",
      "| 12 | nan | what country is <e> in | 1 | which city is in bosnian county | base/aareas/schema/administrative_area_type/subdivides_place | base/aareas/schema/administrative_area_type/pertains_to | bosnian county | bosnian county |\n",
      "| 13 | nan | what classification is <e> | 159 | what classification is the drug sodium sulfacetamide and sulfur 0.05 kit | medicine/drug_formulation/legal_status | medicine/drug_formulation/drug_category | sodium sulfacetamide and sulfur 0.05 kit | sodium sulfacetamide and sulfur 0.05 kit |\n",
      "| 14 | nan | what is <e> 's gender identity | 0 | what is mandy bright's sexual orientation  | base/adultentertainment/adult_entertainer/sexual_orientation | people/person/gender | mandy | mandy bright |\n",
      "| 15 | nan | which releases has <e> as a track ? | 646 | which releases have gold as the track? | music/recording/releases | music/release_track/release | gold | gold |\n",
      "| 16 | nan | what movie was directed by <e> | 1070 | what thriller was directed by Félix Enríquez Alcalá | film/film_genre/films_in_this_genre | film/director/film | thriller | felix enriquez alcala |\n",
      "| 17 | nan | what is the genre of <e> ( film ) | 19 | what's the first episode of the fifth season of deja vu (the outer limits) | tv/tv_series_episode/next_episode | film/film/genre | deja vu | the fifth season of |\n",
      "| 18 | nan | what type of storage does the <e> support | 2 | What type of sensor does the nikon d3 have? | digicams/digital_camera/sensor_type | digicams/digital_camera/supported_storage_types | nikon d3 | nikon d3 |\n",
      "| 19 | nan | what color is <e> | 7 | What color of horse is george washington | base/thoroughbredracing/thoroughbred_racehorse/color | education/educational_institution/colors | george washington | george washington |\n",
      "| 20 | nan | what is the title of a film with the <e> genre ? | 1070 | What is the drama film where black-and-white technique was used? | film/film_genre/films_in_this_genre | film/film_genre/films_in_this_genre | black-and-white | drama film |\n",
      "| 21 | nan | what entity governs the <e> | 123 | what American politician gave her name for the eleanor roosevelt national historic site | symbols/namesake/named_after | protected_sites/protected_site/governing_body | eleanor roosevelt national historic site | eleanor roosevelt national historic |\n",
      "| 22 | nan | what is <e> 's profession ? | 847 | What is aninuddha roy chowdhury's profession? | people/person/profession | common/topic/notable_types | aniruddha roy chowdhury | chowdhury |\n",
      "| 23 | nan | what time of media is <e> | 6 | What type of media is lemon | type/content/media_type | common/topic/notable_types | lemon | lemon |\n",
      "| 24 | nan | what fires were started as a result of <e> | 1 | what scandinavian airlines flight was involved in an accident | aviation/airline/accidents | base/fires/fire_cause/fires_caused_this_way | scandinavian airlines | accident |\n",
      "| 25 | nan | what mountains are located in the <e> | 1243 | what mountains are located within white mountains? | location/location/contains | geography/mountain_range/mountains | white mountains | white mountains |\n",
      "| 26 | nan | what is a <e> group | 727 | what is an electro-industrial genre  | music/genre/albums | music/genre/artists | electro-industrial | electro - industrial |\n",
      "| 27 | nan | what is a <e> program ? | 169 | What is an adventure cvg game? | cvg/cvg_genre/games | tv/tv_genre/programs | adeventure | adventure |\n",
      "| 28 | nan | name a film that was shown at the <e> | 19 | What's a movie that was shown at the 2010 fantasia festival | film/film_festival_event/films | film/film_festival_event/films | 2010 fantasia film festival | 2010 fantasia festival |\n",
      "| 29 | nan | name an <e> person | 1070 | name a 2007 American film parody  | film/film_genre/films_in_this_genre | people/ethnicity/people | parody | american |\n",
      "| 30 | nan | which form of media is <e> | 1 | which department contains caroline bynum | education/academic/departments_old | common/topic/notable_types | caroline bynum | caroline bynum |\n",
      "| 31 | nan | what is one of the tracks off <e> album | 411 | What is one of the songs on the free album? | music/release_track/recording | music/recording/tracks | free | free |\n",
      "| 32 | nan | is <e> pop music or folk | 1553 | is jane child pop music or j-pop | music/artist/genre | music/artist/genre | jane child | jane |\n",
      "| 33 | nan | who is the featured artist on the album <e> ? | 7 | what is the tittle of an entrepreneur who opens and runs restaurants professionally | people/profession/specializations | music/album/featured_artists | entrepreneur | opens |\n",
      "| 34 | nan | what is the country code for <e> | 1 | which country locates japanese city | base/aareas/schema/administrative_area_type/iso_country | base/aareas/schema/administrative_area_type/expected_codes | japanese city | japanese city |\n",
      "| 35 | nan | what does <e> believe | 1610 | what does bangladesh nationalist party identify as | common/topic/notable_types | government/political_party/ideology | bangladesh nationalist party | bangladesh nationalist party |\n",
      "| 36 | nan | which biological process is <e> a part of | 0 | What kind of gas does a nitrogen fixation produce | base/process/process/output | base/change/process/part_of_process | nitrogen fixation | nitrogen fixation |\n",
      "| 37 | nan | what is the name of a woman born in <e> ? | 2 | Who was the father of the famous horse Cincinnati? | biology/organism/parents | location/location/people_born_here | cincinnati | cincinnati |\n",
      "| 38 | nan | what event took place in <e> in 1993 ? | 0 | what event tournament did uganda participate in | sports/sport_country/multi_event_tournaments_participated_in | location/location/events | uganda | uganda |\n",
      "| 39 | nan | what is the genre of <e> . | 1553 | what is the genre of trout fishing in america's work | music/artist/genre | book/book/genre | trout fishing in america | trout fishing in america |\n",
      "| 40 | nan | what album is by <e> | 114 | who boradcasts olivia newton-john | broadcast/artist/content | music/artist/album | olivia newton-john | olivia newton - john |\n",
      "| 41 | nan | where is the movie <e> from | 1374 | where is the movie music for madame from | film/film/country | film/film/country | music for madame | madame |\n",
      "| 42 | nan | where was <e> from | 2720 | where was nat king cole from | people/person/place_of_birth | music/release/region | nat \\\"king\\\" cole | nat king cole |\n",
      "| 43 | nan | what is the ninth track on the album <e> | 679 | travellin light was the fourth track on this album. | music/release/track_list | music/release/track | travellin' light | travellin light |\n",
      "| 44 | nan | what 's a drug with <e> in it | 23 | What's a hydrogen peroxide formula available in liquid | medicine/drug_dosage_form/formulations_available_in_this_form | medicine/drug_ingredient/active_ingredient_of_formulation | liquid | hydrogen peroxide |\n",
      "| 45 | nan | does <e> follow taoism or buddhism | 1 | does gerald j. rip work at the federal court | base/morelaw/canadian_judge/court | people/person/religion | gerald j. rip | gerald j. rip |\n",
      "| 46 | nan | <e> is what gender ? | 1928 | john binder identifies with what gender? | people/person/gender | people/person/gender | john binden | john binder |\n",
      "| 47 | nan | in what release is the track <e> | 454 | bad includes what self-titled track | music/recording/tracks | music/release_track/release | bad | bad |\n",
      "| 48 | nan | which country was <e> made in | 320 | which actress was the daughter of irene born | people/person/children | film/film/country | irene born | the daughter of |\n",
      "| 49 | nan | who was a person in the <e> | 0 | Who's a person killed during the rwandan genocide | base/disaster2/death_causing_event/person_killed | military/military_conflict/military_personnel_involved | rwandan genocide | rwandan genocide |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "import random\n",
    "\n",
    "predicted = []\n",
    "for index, row in tqdm_notebook(df.iterrows(), total=df.shape[0]):\n",
    "    if len(row['candidate_facts']) != 0:\n",
    "        candidate_relations = list(row['candidate_facts'].keys())\n",
    "        kg_relation_probability = [sum(len(row['candidate_facts'][r][mid]) for mid in row['candidate_facts'][r])\n",
    "                                     for r in candidate_relations]\n",
    "        scores = [(row['softmax_ensemble_scores'][i] *\n",
    "                   transform_probability_from_kg_to_train[r] *\n",
    "                   kg_relation_probability[i]) for i, r in enumerate(candidate_relations)]\n",
    "        max_score = max(scores)\n",
    "        predicted_relations = [r for i, r in enumerate(candidate_relations) if scores[i] == max_score]\n",
    "        predicted_relation = random.choice(predicted_relations)\n",
    "        \n",
    "        # We use the `Better than random guessing` from notebook \n",
    "        # `HYPOTHESIS - Question Refers to Multiple Subjects`.\n",
    "        subject_mid, object_mids = sorted(row['candidate_facts'][predicted_relation].items(),\n",
    "                                          key=lambda i: len(i[1]), reverse=True)[0]\n",
    "        predicted.append(tuple([predicted_relation, subject_mid, object_mids]))\n",
    "    else:\n",
    "        predicted.append(tuple([None, None, None]))\n",
    "\n",
    "evaluate(predicted)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}