institution_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download the paper texts (see README.md for details) and store anywhere,\n",
    "# then put the absolute path here. Remember that on windows you have to \n",
    "# type the path as \\\\location\\\\folder\\\\file.txt for example\n",
    "PAPERS_LOCATION='/home/jondea/etymo/workshop/paper_texts/'\n",
    "\n",
    "def load_paper(paper_id):\n",
    "    with open(PAPERS_LOCATION + str(paper_id) +'/fulltext.txt', 'r') as myfile:\n",
    "        data = myfile.read()\n",
    "    \n",
    "    return data\n",
    "\n",
    "import os\n",
    "all_paper_ids = os.listdir(PAPERS_LOCATION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_paper = load_paper(all_paper_ids[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Heuristic 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_chars = ['.', '†', '‡', '¶', '•', '∗', '[', ']', '{', '}', '(', ')']\n",
    "def search_with_simple_context(text, before=20, after=20):\n",
    "    queries = ['University']\n",
    "    i = 0\n",
    "    first_letters = dict()\n",
    "    for s in ['University']:\n",
    "        if s[0] not in first_letters:\n",
    "            first_letters[s[0]] = [s]\n",
    "        else:\n",
    "            first_letters[s[0]].append(s)\n",
    "    \n",
    "    matches_with_context=[]\n",
    "    # Loop through entire text\n",
    "    for c in text:\n",
    "        # Throw out most based on the first character\n",
    "        if c in first_letters:\n",
    "            potential_matches = first_letters[c]\n",
    "            match=None\n",
    "            for potential_match in potential_matches:\n",
    "                j=i+1\n",
    "                match_found=True\n",
    "                for pc in potential_match[1:]:\n",
    "                    if pc != text[j]:\n",
    "                        match_found=False\n",
    "                        break\n",
    "                    j += 1\n",
    "                # Check it is end of word, for example we don't want to pick up \"Label\" as \"Lab\"                     \n",
    "                if not (text[j].isspace() or text[j] in stop_chars):\n",
    "                    match_found = False\n",
    "\n",
    "                if match_found:\n",
    "                    match=potential_match\n",
    "                    break\n",
    "            if match:\n",
    "                context_end = 0\n",
    "                context_start = 0\n",
    "                   \n",
    "                context_end = min(i+len(match)+after,len(text))\n",
    "                context_start = max(i-before,0)\n",
    "                \n",
    "                match_with_context = text[context_start:context_end].strip()\n",
    "\n",
    "                matches_with_context.append(match_with_context)\n",
    "\n",
    "        i += 1\n",
    "\n",
    "    return matches_with_context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['al Sciences, Dalian University of Technology, Dali']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_with_simple_context(example_paper)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Heuristic 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Section title detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The kinds of roman numerals we may see in section titles\n",
    "roman_numerals = ['i', 'v', 'x']\n",
    "\n",
    "# Does it look like a section title?\n",
    "def is_a_section_title(txt, i, section_name):\n",
    "    j = i + len(section_name)\n",
    "    # Apart from whitespace and punctuation is it at the end of the line\n",
    "    while j < len(txt):\n",
    "        c = txt[j]\n",
    "        j += 1\n",
    "        if c == '\\n':\n",
    "            # Found the newline, all is good\n",
    "            break\n",
    "        if c.isspace() or (c in string.punctuation):\n",
    "            # Space or punctuation, keep looking for newline\n",
    "            continue\n",
    "        else:\n",
    "            return False\n",
    "    \n",
    "    # Apart from whitespace, digits and punctuation is it at the start of the line\n",
    "    while i >= 0:\n",
    "        i -= 1\n",
    "        c = txt[i]\n",
    "        if c == '\\n':\n",
    "            return True\n",
    "        if c.isspace() or c.isdigit() or (c in string.punctuation) or c.lower() in roman_numerals:\n",
    "            continue\n",
    "        else:\n",
    "            return False\n",
    "\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'string' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-20741847a4e2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0msection_i3\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m38\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0msection_name3\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'references'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m \u001b[0;32massert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mnot\u001b[0m \u001b[0mis_a_section_title\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msection_title3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msection_i3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msection_name3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m section_title4 = \"\"\"\n",
      "\u001b[0;32m<ipython-input-5-891a17bc5c59>\u001b[0m in \u001b[0;36mis_a_section_title\u001b[0;34m(txt, i, section_name)\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mc\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'\\n'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misspace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misdigit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstring\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpunctuation\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mroman_numerals\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m             \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'string' is not defined"
     ]
    }
   ],
   "source": [
    "# Test this small function\n",
    "section_title1 = \"\"\"\n",
    "s paper.\n",
    "\n",
    "REFERENCES\n",
    "[1] A. K.\n",
    "\"\"\"\n",
    "section_i1=11\n",
    "section_name1='references'\n",
    "assert(is_a_section_title(section_title1, section_i1, section_name1))\n",
    "\n",
    "section_title2 = \"\"\"\n",
    "Jean-Sébastien Gharbi, André Lapied,\n",
    "Bernard Roy, Stéphane Deparis for very helpful comments.\n",
    "\n",
    "References\n",
    "M. Allais. The So-Called Allais Paradox and Rational Decisions under Uncer-\n",
    "tainty. In M. Allais an\n",
    "\"\"\"\n",
    "section_i2=96\n",
    "section_name2='references'\n",
    "assert(is_a_section_title(section_title2, section_i2, section_name2))\n",
    "\n",
    "section_title3 = \"\"\"\n",
    "A paragraph which discusses the word references\n",
    "and has a new line.\n",
    "\"\"\"\n",
    "section_i3=38\n",
    "section_name3='references'\n",
    "assert(not is_a_section_title(section_title3, section_i3, section_name3))\n",
    "\n",
    "section_title4 = \"\"\"\n",
    "\n",
    "6. REFERENCES \n",
    "\n",
    "\"\"\"\n",
    "section_i4=5\n",
    "section_name4='references'\n",
    "assert(is_a_section_title(section_title4, section_i4, section_name4))\n",
    "\n",
    "section_title5 = \"\"\"\n",
    "\n",
    "IV) REFERENCES \n",
    "\n",
    "\"\"\"\n",
    "section_i5=6\n",
    "section_name5='references'\n",
    "assert(is_a_section_title(section_title5, section_i5, section_name5))\n",
    "\n",
    "print(\"Section title detection tests passed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "\n",
    "institution_synonyms = ['University', 'Universiteit', 'Université', 'Universitas',\n",
    "                        'Institute', 'Instituition',\n",
    "                        'Center', 'Centre',\n",
    "                        'Laboratory', 'Labs', 'Lab', 'Laboratoire',\n",
    "                        'Academy', 'Research',\n",
    "                        'Inc', 'Limited', 'PLC', 'LLP', 'GmbH', 'AG']\n",
    "\n",
    "joining_words = ['of', 'the', 'and', 'at', '&', 'for', 'de', 'on', '-']\n",
    "stop_chars = ['.', '†', '‡', '¶', '•', '∗', '[', ']', '{', '}', '(', ')', ',', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n",
    "\n",
    "# Words which make us stop the whole process if we find them on their own\n",
    "kill_words = ['references']\n",
    "\n",
    "# Words which make us throw out the whole institution, these should probably be filtered in the scraper\n",
    "poison_words = ['Proceedings', 'Meeting', 'Annual', 'Quarterly', 'Conference', 'Press', \n",
    "                'Transactions', 'Journal', 'Award', 'pages', 'Projects', 'Fellowship']\n",
    "\n",
    "def is_stop_word(word):\n",
    "    if len(word) == 0:\n",
    "        return True\n",
    "    if word[0].islower() and word not in joining_words:\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "def is_first_letter_of_word(text, i):\n",
    "    if i>0 and text[i-1] in string.ascii_letters:\n",
    "        return False\n",
    "    return True\n",
    "\n",
    "def search_with_context(text, simple_context=False, before=20, after=20):\n",
    "    \n",
    "    institution_first_letters = dict()\n",
    "    for s in institution_synonyms:\n",
    "        if s[0] not in institution_first_letters:\n",
    "            institution_first_letters[s[0]] = [s]\n",
    "        else:\n",
    "            institution_first_letters[s[0]].append(s)\n",
    "            \n",
    "    kill_words_first_letters = dict()\n",
    "    for s in kill_words:\n",
    "        if s[0] not in kill_words_first_letters:\n",
    "            kill_words_first_letters[s[0]] = [s]\n",
    "        else:\n",
    "            kill_words_first_letters[s[0]].append(s)\n",
    "    \n",
    "    matches_with_context=[]\n",
    "\n",
    "    # Loop through entire text character by character\n",
    "    i = -1\n",
    "    for c in text:\n",
    "        i += 1            \n",
    "        \n",
    "        # Kill words stop the whole process\n",
    "        # ---------------------------------\n",
    "        kill = False\n",
    "        if is_first_letter_of_word(text, i) and c.lower() in kill_words_first_letters:\n",
    "            potential_matches = kill_words_first_letters[c.lower()]\n",
    "            for potential_match in potential_matches:\n",
    "                if text[i:i+len(potential_match)].lower() == potential_match:                    \n",
    "                    if is_a_section_title(text, i, potential_match):\n",
    "                        kill = True\n",
    "                        break\n",
    "        # Stop the whole search\n",
    "        if kill:\n",
    "            break\n",
    "        \n",
    "        # Throw out most based on the first character\n",
    "        # Note this is case sensitive because we want University (for example) to be capitalised\n",
    "        if is_first_letter_of_word(text, i) and c in institution_first_letters:\n",
    "            potential_matches = institution_first_letters[c]\n",
    "            match=None\n",
    "            for potential_match in potential_matches:\n",
    "                # Loop through letters starting at the second letter (we've already checked the first)\n",
    "                j=i+1\n",
    "                match_found=True\n",
    "                for pc in potential_match[1:]:\n",
    "                    if pc != text[j]:\n",
    "                        match_found=False\n",
    "                        break\n",
    "                    j += 1\n",
    "                # Check it is end of word, for example we don't want to pick up \"Label\" as \"Lab\"                     \n",
    "                if not (text[j].isspace() or text[j] in stop_chars):\n",
    "                    match_found = False\n",
    "\n",
    "                if match_found:\n",
    "                    match=potential_match\n",
    "                    break\n",
    "\n",
    "            if match:\n",
    "\n",
    "                # Crawl forward\n",
    "                # -------------\n",
    "                context_end = i + len(match)\n",
    "                crawler_i = context_end\n",
    "                current_word = match\n",
    "                stop = False\n",
    "                while context_end < len(text) and not stop:\n",
    "\n",
    "                    # Found a stop char, check the word we have then stop\n",
    "                    if text[crawler_i] in stop_chars:\n",
    "                        stop = True\n",
    "                                        \n",
    "                    if text[crawler_i].isspace() or stop:\n",
    "                        # found a whole word\n",
    "                        if is_stop_word(current_word):\n",
    "                            break\n",
    "                        if len(current_word) > 0 and current_word[0].isupper():\n",
    "                            context_end = crawler_i\n",
    "                        current_word = ''\n",
    "                    else:\n",
    "                        current_word = current_word + text[crawler_i]\n",
    "\n",
    "                    crawler_i += 1\n",
    "\n",
    "                # Crawl backwards\n",
    "                # ---------------\n",
    "                context_start = i\n",
    "                crawler_i = context_start\n",
    "                current_word = match\n",
    "                stop = False\n",
    "                while crawler_i >= 0 and not stop:\n",
    "                    # Found a stop char, check the word we have then stop\n",
    "                    if text[crawler_i] in stop_chars:\n",
    "                        stop = True\n",
    "                    \n",
    "                    if text[crawler_i].isspace() or stop:\n",
    "                        # found a whole word\n",
    "                        if is_stop_word(current_word):\n",
    "                            break\n",
    "                        if len(current_word) > 0 and current_word[0].isupper():\n",
    "                            context_start = crawler_i + 1\n",
    "                        current_word = ''\n",
    "                    else:\n",
    "                        current_word = text[crawler_i] + current_word\n",
    "\n",
    "                    crawler_i -= 1\n",
    "                \n",
    "                match_with_context = text[context_start:context_end].strip()\n",
    "                \n",
    "                # Check if whole string is poisoned (contains a string we know never appears in institutions)\n",
    "                poisoned = False\n",
    "                for poison_word in poison_words:\n",
    "                    if poison_word in match_with_context:\n",
    "                        poisoned = True\n",
    "                        break\n",
    "                \n",
    "                # Remove any which are JUST institution synonyms\n",
    "                trivial = False\n",
    "                if match_with_context in institution_synonyms:\n",
    "                    trivial = True\n",
    "                \n",
    "                if not poisoned and not trivial:\n",
    "                    # Clean up and add\n",
    "                    match_with_context = match_with_context.replace('\\n', ' ')\n",
    "                    matches_with_context.append(match_with_context)\n",
    "\n",
    "    return matches_with_context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_with_context(load_paper(all_paper_ids[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sets = {\n",
    "    37861:['Technische Universiteit Eindhoven', 'University of Pennsylvania'],\n",
    "    38150:['Dalian University of Technology'],\n",
    "    39023:['King Abdullah University of Science and Technology', 'Computational Bioscience Research Center'],\n",
    "    38488:['Indian Institute of Technology Madras'],\n",
    "    37210:['Research Center on Fictitious Economy and Data Science','Kunming University of Science and Technology','University of Chinese Academy of Sciences'],\n",
    "    37435:['Centre for Signal Processing','Hong Kong Polytechnic University','University of Technology, Sydney'],\n",
    "    37835:['Université Paris-Dauphine', 'PSL Research University'],\n",
    "    39572:['Universitas Brawijaya'],\n",
    "    36994:[],\n",
    "    34904:['Erasmus Medical Center', 'IBM Research Shanghai'],\n",
    "    37331:['University of Chinese Academy of Sciences','University of Technology Sydney'],\n",
    "    38300:['University of Electronic Science and Technology of China','Shanghai Institute for Biological Sciences','Chinese Academy of Sciences'],\n",
    "    38732:['ParallelDots, Inc.'],\n",
    "    40424:['University of Maribor', 'Adolf Drolc Healthcare Centre', 'Center for International Cooperation']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "average_score = 0.0\n",
    "counter = 1\n",
    "for paper_id, institutions in test_sets.items():\n",
    "    institutions_found = set(search_with_context(load_paper(paper_id)))\n",
    "    institutions_set = set(institutions)\n",
    "    true_positives = len(institutions_found.intersection(institutions_set))\n",
    "    false_positives =  len(institutions_found - institutions_set)\n",
    "    false_negatives =  len(institutions_set - institutions_found)\n",
    "    \n",
    "    # Calculate f1 score (but with two special cases)\n",
    "    if institutions_found == institutions_set:\n",
    "        # This covers case where the two sets are empty\n",
    "        score = 1.0\n",
    "    elif true_positives == 0:\n",
    "        # Got nothing right\n",
    "        score = 0.0\n",
    "    else:\n",
    "        try:\n",
    "            precision = true_positives / (true_positives + false_positives)\n",
    "            recall = true_positives / (true_positives + false_negatives)\n",
    "            score = 2*(precision*recall)/(precision + recall)\n",
    "        except:\n",
    "            print(\"paper_id:\", paper_id, \"true\", true_positives, \"false positives:\", false_positives, \"false negatives:\", false_negatives, \"score: N/A\",)\n",
    "            continue\n",
    "    average_score = average_score*counter/(counter+1) + score/(counter+1)\n",
    "    counter += 1\n",
    "    print(\"paper_id:\", paper_id, \"true\", true_positives, \"false positives:\",\n",
    "          false_positives, \"false negatives:\", false_negatives, \"score:\", score)\n",
    "\n",
    "print()\n",
    "print(\"Average score:\", average_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Verbose debugging\n",
    "for paper_id, institutions in test_sets.items():\n",
    "    print(\"paper_id\", paper_id)\n",
    "    print(\"True answer\", institutions)\n",
    "    institutions_found = search_with_context(load_paper(paper_id))\n",
    "    print(\"I found\", institutions_found)\n",
    "    print(\"True ones\", set(institutions_found).intersection(set(institutions)))\n",
    "    print(\"I found extra\", set(institutions_found) - set(institutions))\n",
    "    print(\"I missed\", set(institutions) - set(institutions_found))\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}