16_GermEval17-Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('a', 'n/a'),\n",
       " ('a', 'neg'),\n",
       " ('a', 'pos'),\n",
       " ('a', 'neu'),\n",
       " ('b', 'n/a'),\n",
       " ('b', 'neg'),\n",
       " ('b', 'pos'),\n",
       " ('b', 'neu')]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def create_gold_combinations(classes, sentiments):\n",
    "    r = []\n",
    "    for c in classes:\n",
    "        for s in sentiments:\n",
    "            r.append((c, s))\n",
    "            \n",
    "    return r\n",
    "create_gold_combinations(['a', 'b'], ['n/a', 'neg', 'pos', 'neu'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'def create_eval_entries_from_gold_and_predictions(golds, predictions):\n",
    "#     eval_entries = []\n",
    "#     for gold_doc, pred_doc in zip(golds, predictions):\n",
    "#         # only works for the small example where only sentiment is misclassified\n",
    "#         for (g_cls, g_sent), (p_cls, p_sent) in zip(gold_doc, pred_doc):\n",
    "#             if g_cls\n",
    "#         eval_entries.extend(zip(gold_doc, pred_doc))\n",
    "#     return eval_entries\n",
    "\n",
    "# golds = [\n",
    "#     [ # Doc 1\n",
    "#         ('a', 'neg'),\n",
    "#     ], \n",
    "#     [ # Doc 2\n",
    "#         ('a', 'pos'),\n",
    "#         ('b', 'pos')\n",
    "#     ]\n",
    "# ]\n",
    "\n",
    "# predictions = [\n",
    "#     [ # Doc 1\n",
    "#         ('a', 'neg'),\n",
    "#     ], \n",
    "#     [ # Doc 2\n",
    "#         ('a', 'pos'),\n",
    "#         ('b', 'negative')\n",
    "#     ]\n",
    "# ]\n",
    "\n",
    "# eval_entries = create_eval_entries_from_gold_and_predictions(golds, predictions)\n",
    "\n",
    "# eval_entries'\n",
    "eval_entries = [\n",
    "    (('a', 'neu'),('a', 'neu')),\n",
    "    (('a', 'neu'), ('a', 'neu')),\n",
    "    (('b', 'neu'), ('b', 'n/a')),\n",
    "    (('b', 'n/a'), ('b', 'pos'))\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 2, 2, 26)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_metrics(classes, sentiments, eval_entries):\n",
    "    combinations = create_gold_combinations(classes, sentiments)\n",
    "    \n",
    "    tp = 0\n",
    "    fp = 0\n",
    "    tn = 0\n",
    "    fn = 0\n",
    "    \n",
    "    for cls, sent in combinations:\n",
    "        for (g_cls, g_sent), (p_cls, p_sent) in eval_entries:\n",
    "            if g_cls == cls and g_sent == sent:\n",
    "                if g_cls == p_cls and g_sent == p_sent:\n",
    "                    tp += 1\n",
    "                else:\n",
    "                    fn += 1\n",
    "            else:\n",
    "                if p_cls == cls and p_sent == sent:\n",
    "                    fp += 1\n",
    "                else:\n",
    "                    tn += 1\n",
    "                    \n",
    "    return (tp, fp, fn, tn)\n",
    "\n",
    "get_metrics(['a', 'b'], ['n/a', 'neg', 'pos', 'neu'], eval_entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def calculate_micro_f1(classes, sentiments, eval_entries):\n",
    "    tp, fp, fn, _ = get_metrics(classes, sentiments, eval_entries)\n",
    "    if tp == 0 or (tp + fp) == 0 or (tp + fn) == 0:\n",
    "        return 0.0\n",
    "    micro_precission = tp / (tp + fp)\n",
    "    micro_recall = tp / (tp + fn)\n",
    "    \n",
    "    micro_f1 = 2.0 * micro_precission * micro_recall / (micro_precission + micro_recall)\n",
    "    return micro_f1\n",
    "\n",
    "calculate_micro_f1(['a', 'b'], ['n/a', 'neg', 'pos', 'neu'], eval_entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Adaption to tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "from sklearn.metrics import precision_recall_fscore_support, confusion_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PREDICTION:\n",
      "tensor([[0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0]])\n",
      "\n",
      "\n",
      "TARGET:\n",
      "tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
      "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n"
     ]
    }
   ],
   "source": [
    "prediction = torch.LongTensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2],\n",
    "        [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
    "\n",
    "\n",
    "target = torch.LongTensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
    "\n",
    "# per aspect\n",
    "target = torch.t(target)\n",
    "prediction = torch.t(prediction)\n",
    "print('PREDICTION:\\n' + str(prediction))\n",
    "print('\\n\\nTARGET:\\n' + str(target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t Samples\n",
      "Aspect 0 -\t tensor([0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0])\n",
      "Aspect 1 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 2 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 3 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 4 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 5 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 6 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 7 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 8 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 9 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 10 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 11 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 12 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 13 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 14 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 15 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 16 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0])\n",
      "Aspect 17 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 18 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Aspect 19 -\t tensor([0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0])\n"
     ]
    }
   ],
   "source": [
    "print('\\t\\t Samples')\n",
    "for i, aspect in enumerate(prediction):\n",
    "    print(f'Aspect {i} -\\t {aspect}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[((0, 0), (0, tensor(1))),\n",
       " ((0, 0), (0, tensor(1))),\n",
       " ((0, 0), (0, tensor(1))),\n",
       " ((0, 0), (0, tensor(1))),\n",
       " ((0, 0), (0, tensor(1))),\n",
       " ((16, tensor(2)), (16, 0)),\n",
       " ((16, tensor(2)), (16, tensor(2))),\n",
       " ((17, tensor(1)), (17, 0)),\n",
       " ((19, 0), (19, tensor(2)))]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_tensor_eval_entries(prediction, target):\n",
    "    eval_entries = []\n",
    "    # (('a', 'neu'),('a', 'neu')),\n",
    "    for aspect_index, (prediction_aspect, target_aspect) in enumerate(zip(prediction, target)):\n",
    "        #print(f'\\n{aspect_index} Target:\\t{target_aspect}\\n{aspect_index} Prediction\\t{prediction_aspect}')\n",
    "\n",
    "        for y_hat, y in zip(prediction_aspect, target_aspect):\n",
    "                # y is applicable\n",
    "                if y != y_hat and y > 0:\n",
    "                    eval_entries.append(((aspect_index, y), (aspect_index, 0)))\n",
    "\n",
    "                    if y_hat > 0:\n",
    "                        eval_entries.append(((aspect_index, 0), (aspect_index, y_hat)))\n",
    "\n",
    "                elif y == y_hat and y > 0:\n",
    "                    eval_entries.append(((aspect_index, y), (aspect_index, y_hat)))\n",
    "                elif y_hat > 0:\n",
    "                        #print(f'else Pred {y_hat} - Y {y}')\n",
    "                        eval_entries.append(((aspect_index, 0), (aspect_index, y_hat)))\n",
    "\n",
    "\n",
    "    return eval_entries\n",
    "eval_entries = get_tensor_eval_entries(prediction, target)\n",
    "eval_entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 7, 7, 669)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_metrics(range(19), [0, 1, 2, 3], eval_entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.125"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_micro_f1(range(19), [0, 1, 2, 3], eval_entries)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sanity check with example from above"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two samples, Two apsects\n",
    "\n",
    "#### GOLD\n",
    "```xml\n",
    "<Documents>\n",
    "    <Document id=\"1\">\n",
    "        <Opinions>\n",
    "            <Opinion category=\"Allgemein\" polarity=\"neutral\"/>\n",
    "        </Opinions>\n",
    "        <text>COMMENT</text>\n",
    "    </Document>\n",
    "    <Document id=\"2\">\n",
    "        <Opinions>\n",
    "            <Opinion category=\"Allgemein\" polarity=\"neutral\"/>\n",
    "\t\t\t<Opinion category=\"Ticketkauf\" polarity=\"neutral\"/>\n",
    "        </Opinions>\n",
    "        <text>COMMENT</text>\n",
    "    </Document>\n",
    "</Documents>\n",
    "```\n",
    "\n",
    "#### Prediction\n",
    "```xml\n",
    "<Documents>\n",
    "    <Document id=\"1\">\n",
    "        <Opinions>\n",
    "            <Opinion category=\"Allgemein\" polarity=\"neutral\"/>\n",
    "        </Opinions>\n",
    "        <text>COMMENT</text>\n",
    "    </Document>\n",
    "    <Document id=\"2\">\n",
    "        <Opinions>\n",
    "            <Opinion category=\"Allgemein\" polarity=\"neutral\"/>\n",
    "\t\t\t<Opinion category=\"Ticketkauf\" polarity=\"positive\"/>\n",
    "        </Opinions>\n",
    "        <text>COMMENT</text>\n",
    "    </Document>\n",
    "</Documents>\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PREDICTION:\n",
      "tensor([[1., 1.],\n",
      "        [0., 2.]])\n",
      "\n",
      "\n",
      "TARGET:\n",
      "tensor([[1., 1.],\n",
      "        [0., 1.]])\n"
     ]
    }
   ],
   "source": [
    "target = torch.tensor(\n",
    "       [[1., 1.], # aspect a\n",
    "        [0., 1.]  # aspect b\n",
    "       ])\n",
    "\n",
    "prediction = torch.tensor(\n",
    "       [[1., 1.], # aspect a\n",
    "        [0., 2.]  # aspect b\n",
    "       ])\n",
    "\n",
    "\n",
    "# per aspect\n",
    "print('PREDICTION:\\n' + str(prediction))\n",
    "print('\\n\\nTARGET:\\n' + str(target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[((0, tensor(1.)), (0, tensor(1.))),\n",
       " ((0, tensor(1.)), (0, tensor(1.))),\n",
       " ((1, tensor(1.)), (1, 0)),\n",
       " ((1, 0), (1, tensor(2.)))]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_entries = get_tensor_eval_entries(prediction, target)\n",
    "eval_entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 2, 2, 26)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_metrics(range(2), [0, 1, 2, 3], eval_entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_micro_f1(range(19), [0, 1, 2, 3], eval_entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(range(4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Only affecting one aspect at a time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = torch.tensor(\n",
    "       [[1., 1.], # aspect a\n",
    "       ])\n",
    "\n",
    "prediction = torch.tensor(\n",
    "       [[1., 1.], # aspect a\n",
    "       ])\n",
    "eval_entries = get_tensor_eval_entries(prediction, target)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 0, 0, 150)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_metrics(range(19), [0, 1, 2, 3], eval_entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = torch.tensor([[0., 1.]])\n",
    "\n",
    "prediction = torch.tensor([[0., 2.]])\n",
    "eval_entries = get_tensor_eval_entries(prediction, target)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 2, 2, 148)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_metrics(range(19), [0, 1, 2, 3], eval_entries)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Case 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PREDICTION:\n",
      "tensor([[0., 0., 1., 1., 1., 0., 0., 1., 0., 1., 0., 0.],\n",
      "        [2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 2., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 2., 0., 0., 0.]])\n",
      "\n",
      "\n",
      "TARGET:\n",
      "tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [2., 2., 2., 2., 2., 2., 2., 1., 1., 1., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 2., 0., 0., 0., 0., 0., 2., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "prediction = torch.Tensor([[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2],\n",
    "        [1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
    "\n",
    "\n",
    "target = torch.Tensor([[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0],\n",
    "        [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
    "        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])\n",
    "\n",
    "# per aspect\n",
    "target = torch.t(target)\n",
    "prediction = torch.t(prediction)\n",
    "print('PREDICTION:\\n' + str(prediction))\n",
    "print('\\n\\nTARGET:\\n' + str(target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[((0, 0), (0, tensor(1.))),\n",
       " ((0, 0), (0, tensor(1.))),\n",
       " ((0, 0), (0, tensor(1.))),\n",
       " ((0, 0), (0, tensor(1.))),\n",
       " ((0, 0), (0, tensor(1.))),\n",
       " ((1, tensor(2.)), (1, tensor(2.))),\n",
       " ((1, tensor(2.)), (1, tensor(2.))),\n",
       " ((1, tensor(2.)), (1, tensor(2.))),\n",
       " ((1, tensor(2.)), (1, tensor(2.))),\n",
       " ((1, tensor(2.)), (1, tensor(2.))),\n",
       " ((1, tensor(2.)), (1, tensor(2.))),\n",
       " ((1, tensor(2.)), (1, tensor(2.))),\n",
       " ((1, tensor(1.)), (1, 0)),\n",
       " ((1, 0), (1, tensor(2.))),\n",
       " ((1, tensor(1.)), (1, 0)),\n",
       " ((1, 0), (1, tensor(2.))),\n",
       " ((1, tensor(1.)), (1, 0)),\n",
       " ((1, 0), (1, tensor(2.))),\n",
       " ((1, 0), (1, tensor(2.))),\n",
       " ((1, 0), (1, tensor(2.))),\n",
       " ((16, tensor(2.)), (16, 0)),\n",
       " ((16, tensor(2.)), (16, tensor(2.))),\n",
       " ((17, tensor(1.)), (17, 0)),\n",
       " ((19, 0), (19, tensor(2.)))]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_entries = get_tensor_eval_entries(prediction, target)\n",
    "eval_entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8, 16, 16, 1880)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_metrics(range(20), [0, 1, 2, 3], eval_entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3333333333333333"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_micro_f1(range(20), [0, 1, 2, 3], eval_entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'all_predictions.pkl'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-45-46d8b4a0bd53>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'all_predictions.pkl'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mall_predictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mall_predictions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'all_predictions.pkl'"
     ]
    }
   ],
   "source": [
    "with open('all_predictions.pkl', 'rb') as f:\n",
    "    all_predictions = pickle.load(f)\n",
    "    \n",
    "all_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'all_targets.pkl'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-46-5eb6e792e6bd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'all_targets.pkl'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mall_targets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mall_targets\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'all_targets.pkl'"
     ]
    }
   ],
   "source": [
    "with open('all_targets.pkl', 'rb') as f:\n",
    "    all_targets = pickle.load(f)\n",
    "    \n",
    "all_targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'all_predictions' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-47-67711d2be2ba>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mtargets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_predictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_targets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mpredictions\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'all_predictions' is not defined"
     ]
    }
   ],
   "source": [
    "predictions = None\n",
    "targets = None\n",
    "for preds, y in zip(all_predictions, all_targets):\n",
    "    if predictions is None:\n",
    "        predictions = preds\n",
    "        targets = y\n",
    "    else:\n",
    "        predictions = torch.cat((predictions, preds))\n",
    "        targets = torch.cat((targets, y))\n",
    "\n",
    "predictions = torch.t(predictions)\n",
    "targets = torch.t(targets)\n",
    "\n",
    "\n",
    "predictions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "zip argument #1 must support iteration",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-48-baae7b5b71f2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0meval_entries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_tensor_eval_entries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval_entries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-28-23990256a1f5>\u001b[0m in \u001b[0;36mget_tensor_eval_entries\u001b[0;34m(prediction, target)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0meval_entries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;31m# (('a', 'neu'),('a', 'neu')),\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0maspect_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mprediction_aspect\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_aspect\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprediction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m         \u001b[0;31m#print(f'\\n{aspect_index} Target:\\t{target_aspect}\\n{aspect_index} Prediction\\t{prediction_aspect}')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: zip argument #1 must support iteration"
     ]
    }
   ],
   "source": [
    "eval_entries = get_tensor_eval_entries(predictions, targets)\n",
    "len(eval_entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8, 16, 16, 1880)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_metrics(range(20), [0, 1, 2, 3], eval_entries)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean micro f1: 0.14301008196206047\n"
     ]
    }
   ],
   "source": [
    "mean_score = 0.0\n",
    "for i in range(1000):\n",
    "    prediction = torch.randint(0, 4, (12, 20))\n",
    "    prediction = torch.t(prediction)\n",
    "    target = torch.randint(0, 4, (12, 20))\n",
    "    target = torch.t(target)\n",
    "    eval_entries = get_tensor_eval_entries(prediction, target)\n",
    "    mean_score += calculate_micro_f1(range(20), [0, 1, 2, 3], eval_entries)\n",
    "print(f'Mean micro f1: {mean_score/1000}')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}