notebooks/results-extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results Extraction\n",
    "\n",
    "This notebook presents how to use AxCell for retrieval of machine learning results."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the pipeline to work we need a running elasticsearch instance. Run `docker-compose up -d` from the `axcell` repository to start a new instance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from axcell.data.paper_collection import PaperCollection\n",
    "from pathlib import Path\n",
    "\n",
    "ROOT_PATH = Path('data')\n",
    "PAPERS_PATH = ROOT_PATH / 'papers'\n",
    "pc = PaperCollection.from_files(PAPERS_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download and unpack the archive with trained models (table type classifier, table segmentation), taxonomy and abbreviations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODELS_URL = 'https://github.com/paperswithcode/axcell/releases/download/v1.0/models.tar.xz'\n",
    "MODELS_ARCHIVE = 'models.tar.xz'\n",
    "MODELS_PATH = Path('models')\n",
    "\n",
    "from fastai.core import download_url\n",
    "import tarfile\n",
    "\n",
    "download_url(MODELS_URL, MODELS_ARCHIVE)\n",
    "with tarfile.open(MODELS_ARCHIVE, 'r:*') as archive:\n",
    "    archive.extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[PID 21480] Load model table-structure-classifier.pth\n"
     ]
    }
   ],
   "source": [
    "from axcell.helpers.results_extractor import ResultsExtractor\n",
    "\n",
    "extract_results = ResultsExtractor(MODELS_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "paper = pc.get_by_id('1903.11816')\n",
    "results = extract_results(paper, in_place=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>metric</th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PASCAL Context</td>\n",
       "      <td>mIoU</td>\n",
       "      <td>Semantic Segmentation</td>\n",
       "      <td>EncNet+JPU (ours)</td>\n",
       "      <td>53.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADE20K</td>\n",
       "      <td>Validation mIoU</td>\n",
       "      <td>Semantic Segmentation</td>\n",
       "      <td>Ours</td>\n",
       "      <td>80.99</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dataset           metric                   task              model  \\\n",
       "0  PASCAL Context             mIoU  Semantic Segmentation  EncNet+JPU (ours)   \n",
       "1          ADE20K  Validation mIoU  Semantic Segmentation               Ours   \n",
       "\n",
       "   score  \n",
       "0  53.10  \n",
       "1  80.99  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With `in_place=True` we can inspect the inferred segmentation of the tables predicted to be a leaderboard or ablation tables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       "body{margin:0;padding:0;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}code{font-family:source-code-pro,Menlo,Monaco,Consolas,Courier New,monospace}.tableWrapper{-overflow:auto}.tableWrapper .has-annotations{color:#ff3860}.tableWrapper .model-params{background-color:#209cee;color:rgba(0,0,0,.7)}.tableWrapper .table-meta{background-color:#fff3c5;color:rgba(0,0,0,.7)}.tableWrapper .model-best{background-color:#ff3860;color:rgba(0,0,0,.7)}.tableWrapper .model-ensemble{background-color: #aa38ff;color: #fff;}.tableWrapper .model-competing{background-color:#ffdd57;color:rgba(0,0,0,.7)}.tableWrapper .model-paper{background-color:#ff3860;color:#fff}.tableWrapper .dataset-sub{background-color:#23d160;color:#fff}.tableWrapper .dataset-metric{background-color:#209cee;color:#fff}.tableWrapper .dataset{background-color:#02bd43;color:#fff}.tableWrapper .trash{background-color:#363636;color:#f5f5f5}.tableWrapper .wtf{background-color:#f0f;color:#f5f5f5}.tableWrapper .dataset-task{background-color:#77ecdd;color:rgba(0,0,0,.7)}.tableWrapper .dataset-paper{background-color:#e4ffee;color:rgba(0,0,0,.7)}.tableWrapper td.focused-cell{outline:2px solid #9ecaed;border-radius:7px;box-shadow:0 0 10px #9ecaed}.tableWrapper span.text-bold{font-weight:700}.tableWrapper span.text-italic{font-style:italic}.tableWrapper span.text-red{color:red}.tableWrapper span.text-green{color:green}.tableWrapper span.text-blue{color:#00f}.predict-dataset,.predict-dataset-metric,.predict-model-competing,.predict-model-paper,.predict-model-params,.predict-table-meta{outline:2px solid #9ecaed;border-radius:7px;box-shadow:0 0 10px #9ecaed}.tableWrapper .predict-model-params{background-color:#209cee;color:rgba(0,0,0,.7)}.tableWrapper .predict-table-meta{background-color:#fff3c5;color:rgba(0,0,0,.7)}.tableWrapper .predict-model-competing{background-color:#ffdd57;color:rgba(0,0,0,.7)}.tableWrapper .predict-model-paper{background-color:#ff3860;color:#fff}.tableWrapper .predict-dataset-metric{background-color:#209cee;color:#fff}.tableWrapper .predict-dataset{background-color:#02bd43;color:#fff}.tableWrapper td{border:inherit}.tableWrapper table tr td.border-l{border-left:1px solid #000}.tableWrapper table tr td.border-r{border-right:1px solid #000}.tableWrapper table tr td.border-t{border-top:1px solid #000}.tableWrapper table tr td.border-b{border-bottom:1px solid #000}.tableWrapper table tr td.border-ll{border-left:2px solid #000}.tableWrapper table tr td.border-rr{border-right:2px solid #000}.tableWrapper table tr td.border-tt{border-top:2px solid #000}.tableWrapper table tr td.border-bb{border-bottom:2px solid #000}.tableWrapper table tr td.align-left{text-align:left}.tableWrapper table tr td.align-right{text-align:right}.tableWrapper table tr td.align-center{text-align:center}.tableWrapper table tr td.align-justify{text-align:justify}div.form-group>input.form-control.input-sm{border-radius:2px;font-size:.75rem;background-color:#fff;color:#363636;box-shadow:inset 0 1px 2px rgba(10,10,10,.1);max-width:100%;width:100%;height:2.25em;padding:calc(.375em - 1px) calc(.625em - 1px);position:relative;border:1px solid #b5b5b5}div.form-group>input.form-control.input-sm:focus{border-color:#3273dc;box-shadow:0 0 0 .125em rgba(50,115,220,.25)}.ht_clone_top{z-index:20}.evaluation-tables{overflow:scroll;max-height:20vh;border-top:1px solid #a9a9a9}.navbar.is-fixed-bottom,.navbar.is-fixed-top{z-index:200}body{padding-bottom:20vh}\n",
       ".tableWrapper .final-proposal{ background: lightgreen }\n",
       "</style>\n",
       "\n",
       "<div class=\"tableWrapper\">\n",
       "<table>\n",
       "<tr>\n",
       "<td class=\"table-meta border-r border-t align-right header \" title=\"\">Method</td>\n",
       "<td class=\"table-meta border-r border-t align-left header \" title=\"\">Backbone</td>\n",
       "<td class=\"dataset-metric border-t align-center header \" title=\"\">mIoU%</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r border-t align-right \" title=\"\">FCN-8s [<a title=\"bib-bib22\">22</a>]</td>\n",
       "<td class=\" border-r border-t \" title=\"\"></td>\n",
       "<td class=\" border-t align-center \" title=\"\">37.8</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r align-right \" title=\"\">CRF-RNN [<a title=\"bib-bib39\">39</a>]</td>\n",
       "<td class=\" border-r \" title=\"\"></td>\n",
       "<td class=\" align-center \" title=\"\">39.3</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r align-right \" title=\"\">ParseNet [<a title=\"bib-bib21\">21</a>]</td>\n",
       "<td class=\" border-r \" title=\"\"></td>\n",
       "<td class=\" align-center \" title=\"\">40.4</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r align-right \" title=\"\">BoxSup [<a title=\"bib-bib10\">10</a>]</td>\n",
       "<td class=\" border-r \" title=\"\"></td>\n",
       "<td class=\" align-center \" title=\"\">40.5</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r align-right \" title=\"\">HO_CRF [<a title=\"bib-bib2\">2</a>]</td>\n",
       "<td class=\" border-r \" title=\"\"></td>\n",
       "<td class=\" align-center \" title=\"\">41.3</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r align-right \" title=\"\">Piecewise [<a title=\"bib-bib19\">19</a>]</td>\n",
       "<td class=\" border-r \" title=\"\"></td>\n",
       "<td class=\" align-center \" title=\"\">43.3</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r align-right \" title=\"\">VeryDeep [<a title=\"bib-bib32\">32</a>]</td>\n",
       "<td class=\" border-r \" title=\"\"></td>\n",
       "<td class=\" align-center \" title=\"\">44.5</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r align-right \" title=\"\">DeepLabV2 [<a title=\"bib-bib5\">5</a>]</td>\n",
       "<td class=\"table-meta border-r align-left \" title=\"\">ResNet-101 + COCO</td>\n",
       "<td class=\" align-center \" title=\"\">45.7</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r align-right \" title=\"\">RefineNet [<a title=\"bib-bib18\">18</a>]</td>\n",
       "<td class=\"table-meta border-r align-left \" title=\"\">ResNet-152</td>\n",
       "<td class=\" align-center \" title=\"\">47.3</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r align-right \" title=\"\">EncNet [<a title=\"bib-bib36\">36</a>]</td>\n",
       "<td class=\"table-meta border-r align-left \" title=\"\">ResNet-101</td>\n",
       "<td class=\" align-center \" title=\"\">51.7</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-competing border-r align-right \" title=\"\">DUpsampling [<a title=\"bib-bib29\">29</a>]</td>\n",
       "<td class=\"table-meta border-r align-left \" title=\"\">Xception-71</td>\n",
       "<td class=\" align-center \" title=\"\">52.5</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-best border-r border-t align-right \" title=\"\">EncNet+JPU (ours)</td>\n",
       "<td class=\"table-meta border-r border-t align-left \" title=\"\">ResNet-50</td>\n",
       "<td class=\" border-t align-center \" title=\"\">51.2</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td class=\"model-best border-b border-r align-right \" title=\"\">EncNet+JPU (ours)</td>\n",
       "<td class=\"table-meta border-b border-r align-left \" title=\"\">ResNet-101</td>\n",
       "<td class=\" border-b align-center \" title=\"\"><span class=\"text-bold\">53.1</span></td>\n",
       "</tr>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "<axcell.data.table.Table at 0x7f59ce044e10>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paper.tables[2]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}