process_CL-SciSumm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk \n",
    "import glob, os\n",
    "import re\n",
    "import string\n",
    "import sys\n",
    "import random\n",
    "import xml.etree.ElementTree as ET\n",
    "import xml.etree\n",
    "import numpy\n",
    "\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from collections import Counter\n",
    "from nltk.corpus import stopwords\n",
    "import html\n",
    "from lxml import etree\n",
    "import unidecode\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/chryssa/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/chryssa/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package wordnet to /home/chryssa/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/chryssa/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def checkQuality(sentence):\n",
    "    \n",
    "    lemmatizer = WordNetLemmatizer() \n",
    "    tokenized_text = nltk.word_tokenize(sentence)\n",
    "    length = len(tokenized_text)\n",
    "    alpha = 0\n",
    "    nonAlpha = 0\n",
    "    found = 0\n",
    "    nonFound = 0\n",
    "    upper = 0\n",
    "    stop = 0\n",
    "    stop_words = set(stopwords.words('english')) \n",
    "    \n",
    "    for token in tokenized_text:\n",
    "        #print(token)\n",
    "        if len(token)==1:\n",
    "            if token.isalpha():\n",
    "                alpha+=1\n",
    "            else:\n",
    "                nonAlpha+=1\n",
    "        else:\n",
    "            if(token.lower() in stop_words):\n",
    "                stop+=1\n",
    "            else:\n",
    "                lemma = lemmatizer.lemmatize(token.lower())\n",
    "                if len(wn.synsets(lemma))>0:\n",
    "                    found+=1\n",
    "                else:\n",
    "                    if token.isalpha():\n",
    "                        if not token.islower() and not token.isupper():\n",
    "                            upper+=1\n",
    "                            #print(token)\n",
    "                        elif token.isupper():\n",
    "                            upper+=1\n",
    "                        else:\n",
    "                            nonFound+=1\n",
    "                    else:\n",
    "                        nonAlpha+=1\n",
    "    alphaS = alpha/length\n",
    "    nonAlphaS = nonAlpha/length\n",
    "    foundS = found/length\n",
    "    nonFoundS = nonFound/length\n",
    "    upperS = upper/length\n",
    "    stopS = stop/length\n",
    "    nv = False\n",
    "    if(nonFoundS>0.3):\n",
    "        #print(sentence + \"====>NF:\" + str(nonFound))\n",
    "        nv = True\n",
    "    if(foundS<0.11):\n",
    "        #print(sentence + \"====>F:\" + str(foundS))\n",
    "        nv = True\n",
    "    if(alphaS>0.2):\n",
    "        #print(sentence + \"====>AF:\" + str(alphaS))\n",
    "        nv = True\n",
    "    if nv:\n",
    "        return False\n",
    "    return True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filtering(senText, buffSent):\n",
    "    mat = False\n",
    "    p = re.compile('\\([a-zA-Z]\\)')\n",
    "    #print(senText)\n",
    "    re.sub(' +', ' ', senText)\n",
    "    \n",
    "    m = p.match(senText)\n",
    "\n",
    "    if m:\n",
    "        if m.start() == 0:\n",
    "            mat = True\n",
    "    if senText.strip().startswith(\"(\") and not mat:\n",
    "        if buffSent.endswith(\".\"):\n",
    "            return True\n",
    "    elif (buffSent.strip().endswith(\". . .\") or buffSent.strip().endswith(\"...\")):\n",
    "        return True\n",
    "    elif not senText[0].isupper() and not mat:\n",
    "        if buffSent.strip().endswith(\")\"):\n",
    "            return True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parseSentXMLtree(xmlTree, offsets):\n",
    "    buffSent = \"\"\n",
    "    buffSid = []\n",
    "    buffSsid = []\n",
    "    entries = []\n",
    "\n",
    "    for sent in xmlTree.iter(\"S\"):\n",
    "        mat = False\n",
    "        fixed = False\n",
    "        sid = sent.attrib['sid']\n",
    "        if sid == \"0\" or sid ==\"\":\n",
    "            ssid = \"0\"\n",
    "        else:\n",
    "            ssid = sent.attrib['ssid']\n",
    "        senText = sent.text\n",
    "       \n",
    "        if(len(offsets)==1):\n",
    "            entries.append([senText, sid, ssid])\n",
    "\n",
    "        else:\n",
    "            #fixing\n",
    "            if senText!=None:\n",
    "\n",
    "                fixed = filtering(senText, buffSent)\n",
    "                if (not fixed) and (len(buffSid)>0):\n",
    "                    entries.append([buffSent, str(buffSid), str(buffSsid)]) \n",
    "                    buffSid.clear()\n",
    "                    buffSsid.clear()\n",
    "                    buffSent = \"\"\n",
    "                \n",
    "                buffSid.append(sid)\n",
    "                buffSsid.append(ssid)\n",
    "                buffSent+=senText\n",
    "\n",
    "    if len(offsets)>1:\n",
    "        entries.append([buffSent, str(buffSid), str(buffSsid)])  \n",
    "\n",
    "    buffSid.clear()\n",
    "    buffSsid.clear()\n",
    "    buffSent = \"\"\n",
    "    \n",
    "    return entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "### parse reference sentence xmls to be used for generation of negative pairs ###\n",
    "\n",
    "def parse_references(year='2019',docType='Reference'):\n",
    "\n",
    "    filenames =[]\n",
    "    fileCounter = 0\n",
    "    p = re.compile('\\([a-zA-Z]\\)')\n",
    "    initDir = \"CL-SciSumm/scisumm-corpus-master/data/Training-Set-\"+year+\"/\"\n",
    "    writeDir = \"CL-SciSumm/scisumm-corpus-master/data/\"+docType +\"_Sent-\"+year+\"/\"\n",
    "    writeDir2 = \"CL-SciSumm/scisumm-corpus-master/data/\"+docType +\"_Sent-\"+year+\"_plain/\"\n",
    "\n",
    "    if not os.path.exists(writeDir):\n",
    "        os.makedirs(writeDir)\n",
    "    if not os.path.exists(writeDir2):\n",
    "        os.makedirs(writeDir2)\n",
    "\n",
    "    for root, dirs, files in os.walk(initDir):\n",
    "        dir = os.path.basename(root)\n",
    "        if dir == docType + \"_XML\":\n",
    "            for file in files:\n",
    "                if file.endswith(\".xml\"):\n",
    "                    fileStr=str(root)\n",
    "                    fileStr+=\"/\"\n",
    "                    fileStr+=file\n",
    "                    #print(fileStr)\n",
    "                    if not file in filenames:\n",
    "                        filenames.append(file)\n",
    "                        with open(fileStr, encoding='utf-8', errors='ignore') as xml2str:\n",
    "                            data=xml2str.read()\n",
    "                            parser = ET.XMLParser()\n",
    "                            xmlTree = ET.fromstring(data,parser)\n",
    "                            buffSent = \"\"\n",
    "                            buffSid = []\n",
    "                            buffSsid = []\n",
    "\n",
    "\n",
    "                            wf = open(writeDir+file.split(\".\")[0]+\"_sentences.txt\", \"w\") \n",
    "                            wf2 = open(writeDir2+file.split(\".\")[0]+\"_sentences_plain.txt\", \"w\") \n",
    "                            fileCounter+=1\n",
    "                            for sent in xmlTree.iter(\"S\"):\n",
    "                                mat = False\n",
    "                                fixed = False\n",
    "                                sid = sent.attrib['sid']\n",
    "                                if sid == \"0\" or sid ==\"\":\n",
    "                                    ssid = \"0\"\n",
    "                                else:\n",
    "                                    ssid = sent.attrib['ssid']\n",
    "                                senText = sent.text\n",
    "\n",
    "                                #fix parsing\n",
    "                                if senText!=None:\n",
    "                                    fixed = filtering(senText,buffSent)\n",
    "                                else:\n",
    "                                    fixed = False\n",
    "                                if (not fixed) and (len(buffSid)>0):\n",
    "                                    wf.write(buffSent+\"\\t\"+str(buffSid)+\"\\t\"+str(buffSsid)+\"\\n\") \n",
    "                                    wf2.write(buffSent+\"\\n\") \n",
    "                                    buffSid.clear()\n",
    "                                    buffSsid.clear()\n",
    "                                    buffSent = \"\"\n",
    "                                if senText!=None:   \n",
    "                                    buffSid.append(sid)\n",
    "                                    buffSsid.append(ssid)\n",
    "                                    buffSent+=senText\n",
    "\n",
    "                            wf.close()\n",
    "    print(fileCounter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "parse_references('2019','Reference')\n",
    "parse_references('2018','Reference')\n",
    "parse_references('2018','Citance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generate positive pairs \n",
    "def generate_positive_pairs(year):\n",
    "\n",
    "\n",
    "    initDir = \"CL-SciSumm/scisumm-corpus-master/From-ScisummNet-\"+year+\"/\"\n",
    "    annotationFile = \".ann\"\n",
    "    counter = 0;\n",
    "    paircount = 0;\n",
    "    writeDir1 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_offsets_pos-\"+year+\"/\"\n",
    "    writeDir2 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_pos-\"+year+\"/\"\n",
    "    writeDir3 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_offsets_full_pos-\"+year+\"/\"\n",
    "    writeDir4 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_full_pos-\"+year+\"/\"\n",
    "\n",
    "    if not os.path.exists(writeDir1):\n",
    "        os.makedirs(writeDir1)\n",
    "    if not os.path.exists(writeDir2):\n",
    "        os.makedirs(writeDir2)\n",
    "    if not os.path.exists(writeDir3):\n",
    "        os.makedirs(writeDir3)\n",
    "    if not os.path.exists(writeDir4):\n",
    "        os.makedirs(writeDir4)\n",
    "\n",
    "    for root, dirs, files in os.walk(initDir):\n",
    "        for file in files:    \n",
    "             if annotationFile in file:\n",
    "                with open (os.path.join(root, file), \"r\") as annFile:\n",
    "                    counter+=1\n",
    "                    wf1 = open(writeDir1+file.split(\".\")[0]+\"_pairs_off.tsv\", \"w\") \n",
    "                    wf2 = open(writeDir2+file.split(\".\")[0]+\"_pairs.tsv\", \"w\")    \n",
    "                    wf3 = open(writeDir3+file.split(\".\")[0]+\"_pairs_off.tsv\", \"w\") \n",
    "                    wf4 = open(writeDir4+file.split(\".\")[0]+\"_pairs.tsv\", \"w\")    \n",
    "                    for line in annFile:\n",
    "                        if(len(line)>1):\n",
    "                            xmlCitText = \"\"\n",
    "                            xmlRefText = \"\"\n",
    "\n",
    "                            cit_num = \"0\"\n",
    "                            fields  = line.split(\" | \")\n",
    "                            for field in fields:\n",
    "                                if field.startswith(\"Citance Number:\"):\n",
    "                                    start = field.find(\":\")\n",
    "                                    cit_num = field[start+1:].strip()\n",
    "                                elif(field.startswith(\"Reference Article:\")):\n",
    "                                    start = field.find(\":\")\n",
    "                                    if(\".\" in field):\n",
    "                                        end = field.find(\".\", start)\n",
    "                                        name_R = field[start+1:end].strip()\n",
    "                                    else:\n",
    "                                         name_R = field[start+1:].strip()\n",
    "                                elif field.startswith(\"Citing Article:\"):\n",
    "                                    start = field.find(\":\") \n",
    "                                    if(\".\" in field):\n",
    "                                        end = field.find(\".\", start)\n",
    "                                        name_C = field[start+1:end].strip()\n",
    "                                    else:\n",
    "                                         name_C = field[start+1:].strip()\n",
    "                                elif field.startswith(\"Citation Offset\"):\n",
    "                                    start = field.find(\"\\[\") \n",
    "                                    end = field.find(\"\\]\", start)\n",
    "                                    citOffset = field[start+1:end]\n",
    "                                    citOffsets = citOffset.split(\",\")\n",
    "                                elif field.startswith(\"Citation Text:\"):\n",
    "                                    start = field.find(\":\") \n",
    "                                    end = field.find(\">\", start)\n",
    "                                    xmlCitText = field[start+1:].strip()\n",
    "                                elif field.startswith(\"Reference Offset\"):\n",
    "                                    start = field.find(\"\\[\") \n",
    "                                    end = field.find(\"\\]\", start)\n",
    "                                    refOffset = field[start+1:end]\n",
    "                                    refOffsets = refOffset.split(\",\")\n",
    "                                elif field.startswith(\"Reference Text:\"):\n",
    "                                    start = field.find(\":\") \n",
    "                                    xmlRefText = field[start+1:].strip()\n",
    "\n",
    "                            parser1 = etree.XMLParser(recover=True)\n",
    "                            parser2 = etree.XMLParser(recover=True)\n",
    "\n",
    "                            try:\n",
    "                                xmlEntryCit = \"<root>\"+xmlCitText.replace(\" sid = \",\" sid =\\\"0\\\"\")+\"</root>\"\n",
    "\n",
    "                                xmlCitTree = ET.fromstring(xmlEntryCit.encode('UTF-8', 'ignore'),parser1)\n",
    "                            except:\n",
    "                                print(\"Except:\"+xmlEntryCit)\n",
    "                                print(\"\\n\"+str(counter))\n",
    "                                raise\n",
    "\n",
    "                            try:\n",
    "                                xmlEntryRef=\"<root> \"+xmlRefText.replace(\" sid = \",\" sid =\\\"\").replace(\" ssid =\",\"\\\" ssid =\").replace(\"ssid = \",\"ssid = \\\"0\\\"\").replace(\"&\", \"\\&\")+\" </root>\"\n",
    "\n",
    "                                xmlRefTree = ET.fromstring(xmlEntryRef.encode('utf-8', 'ignore'),parser2)\n",
    "                            except:\n",
    "                                print(\"Except2:\"+xmlEntryRef)\n",
    "                                print(\"\\n\"+str(counter))\n",
    "                                raise\n",
    "\n",
    "\n",
    "                            citEntries = parseSentXMLtree(xmlCitTree, citOffsets)\n",
    "\n",
    "                            refEntries = parseSentXMLtree(xmlRefTree, refOffsets)\n",
    "\n",
    "                            for citEntry in citEntries:\n",
    "\n",
    "                                if not str(citEntry[0])==\"None\":\n",
    "                                    for refEntry in refEntries:\n",
    "\n",
    "                                        pair_off = citEntry[0] + \"\\t\" + refEntry[0] + \"\\t\" +  \"\\t\" +  \"\\t\" + str(refEntry[1]) + \"\\t\" + \"\\t\" + cit_num + \"\\t\" + name_C\n",
    "                                        pair = citEntry[0] + \"\\t\" + refEntry[0]\n",
    "                                        paircount = paircount+1\n",
    "                                        wf1.write(pair_off + \"\\n\")\n",
    "                                        wf2.write(pair + \"\\n\")\n",
    "\n",
    "\n",
    "                            all_cit = \"\"\n",
    "                            all_sid = []\n",
    "                            all_ssid = []\n",
    "                            if len(citEntries)>1:\n",
    "                                print(citEntries)\n",
    "                                for citEntry in citEntries:\n",
    "                                    all_cit+=citEntry[0]+\" \"\n",
    "                                    all_sid.append(citEntry[1])\n",
    "                                    all_ssid.append(citEntry[1])\n",
    "                                all_cit = all_cit.strip()\n",
    "                                for refEntry in refEntries:\n",
    "                                    pair_off = all_cit + \"\\t\" + refEntry[0] + \"\\t\" +  \"\\t\" + \"\\t\" + str(refEntry[1]) + \"\\t\" + cit_num + \"\\t\" + name_C\n",
    "                                    pair = all_cit + \"\\t\" + refEntry[0] \n",
    "                                    wf3.write(pair_off + \"\\n\")\n",
    "                                    wf4.write(pair + \"\\n\")   \n",
    "                            citEntries.clear()\n",
    "                            refEntries.clear()\n",
    "    print(paircount)         \n",
    "    print(counter)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_positive_pairs('2018')\n",
    "generate_positive_pairs('2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "196228\n",
      "40\n"
     ]
    }
   ],
   "source": [
    "#process negative pairs for CL-SciSumm manually annotated data (2018)\n",
    "#40 docs\n",
    "#generate all possible pairs (exhaustive)\n",
    "\n",
    "\n",
    "year = \"2018\"\n",
    "initDir = \"CL-SciSumm/scisumm-corpus-master/data/Training-Set-\"+year+\"/\"\n",
    "annotationFile = \".ann\"\n",
    "\n",
    "\n",
    "counter = 0;\n",
    "paircount = 0;\n",
    "\n",
    "writeDir1 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_offsets_corr_exhaustive_neg-\"+year+\"/\"\n",
    "writeDir2 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_corr_exhaustive_neg-\"+year+\"/\"\n",
    "writeDir3 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_offsets_full_corr_exhaustive_neg-\"+year+\"/\"\n",
    "writeDir4 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_full_corr_exhaustive_neg-\"+year+\"/\"\n",
    "\n",
    "if not os.path.exists(writeDir1):\n",
    "    os.makedirs(writeDir1)\n",
    "if not os.path.exists(writeDir2):\n",
    "    os.makedirs(writeDir2)\n",
    "if not os.path.exists(writeDir3):\n",
    "    os.makedirs(writeDir3)\n",
    "if not os.path.exists(writeDir4):\n",
    "    os.makedirs(writeDir4)\n",
    "for root, dirs, files in os.walk(initDir):\n",
    "    for file in files:    \n",
    "        if annotationFile in file:\n",
    "             with open (os.path.join(root, file), \"r\") as annFile:\n",
    "                counter+=1\n",
    "               \n",
    "                wf1 = open(writeDir1+file.split(\".\")[0]+\"_pairs_off.tsv\", \"w\") \n",
    "                wf2 = open(writeDir2+file.split(\".\")[0]+\"_pairs.tsv\", \"w\")    \n",
    "                wf3 = open(writeDir3+file.split(\".\")[0]+\"_pairs_off.tsv\", \"w\") \n",
    "                wf4 = open(writeDir4+file.split(\".\")[0]+\"_pairs.tsv\", \"w\")    \n",
    "                \n",
    "                for line in annFile:\n",
    "                    if(len(line)>1):\n",
    "                        xmlCitText = \"\"\n",
    "                        xmlRefText = \"\"\n",
    "                        \n",
    "                        cit_num = \"0\"\n",
    "                        fields  = line.split(\" | \")\n",
    "                        for field in fields:\n",
    "                            if field.startswith(\"Citance Number:\"):\n",
    "                                start = field.find(\":\")\n",
    "                                cit_num = field[start:].strip()\n",
    "                            elif(field.startswith(\"Reference Article:\")):\n",
    "                                start = field.find(\":\")\n",
    "                                if(\".\" in field):\n",
    "                                    end = field.find(\".\", start)\n",
    "                                    name_R = field[start+1:end].strip()\n",
    "                                else:\n",
    "                                     name_R = field[start+1:].strip()\n",
    "                            elif field.startswith(\"Citing Article:\"):\n",
    "                                start = field.find(\":\") \n",
    "                                if(\".\" in field):\n",
    "                                    end = field.find(\".\", start)\n",
    "                                    name_C = field[start+1:end].strip()\n",
    "                                else:\n",
    "                                     name_C = field[start+1:].strip()\n",
    "                            elif field.startswith(\"Citation Offset\"):\n",
    "                                start = field.find(\"\\[\") \n",
    "                                end = field.find(\"\\]\", start)\n",
    "                                citOffset = field[start+1:end]\n",
    "                                citOffsets = citOffset.split(\",\")\n",
    "                            elif field.startswith(\"Citation Text:\"):\n",
    "                                start = field.find(\":\") \n",
    "                                end = field.find(\">\", start)\n",
    "                                xmlCitText = field[start+1:].strip()\n",
    "                            elif field.startswith(\"Reference Offset\"):\n",
    "                                start = field.find(\"\\[\") \n",
    "                                end = field.find(\"\\]\", start)\n",
    "                                refOffset = field[start+1:end]\n",
    "                                refOffsets = refOffset.split(\",\")\n",
    "                            elif field.startswith(\"Reference Text:\"):\n",
    "                                start = field.find(\":\") \n",
    "                                xmlRefText = field[start+1:].strip()\n",
    "                        \n",
    "                        #print(name_C)\n",
    "                        \n",
    "                        refer_sentences_file = \"CL-SciSumm/scisumm-corpus-master/data/Reference_Sent-\"+year+\"/\"+name_R+\"_sentences.txt\"\n",
    "                   \n",
    "                        lines = []\n",
    "                        negSentences= []\n",
    "                        if(os.path.isfile(refer_sentences_file)):\n",
    "                            with open(refer_sentences_file) as f:\n",
    "                                lines = f.read().splitlines()\n",
    "                            max = len(lines)-1\n",
    "                            min = 0\n",
    "                            neg = [];\n",
    "                            \n",
    "                            for lineNum in range(1,len(lines)):\n",
    "                                \n",
    "                                candidate = lines[lineNum]\n",
    "                                #print(candidate.split(\"\\t\")[1])\n",
    "                                offsets = candidate.split(\"\\t\")[1].strip(\"\\[\\]\").split(\",\")\n",
    "                                valid = True\n",
    "                                if not lineNum in neg:\n",
    "                                    for offset in offsets:\n",
    "                                        if offset in refOffsets:\n",
    "                                             valid = False\n",
    "                                    qual = False\n",
    "                                    if valid:\n",
    "                                        qual = checkQuality(candidate.split(\"\\t\")[0])\n",
    "                                    if valid and qual:\n",
    "                                        neg.append(lineNum)\n",
    "                                        negSentences.append(candidate)\n",
    "                                        \n",
    "                        else:\n",
    "                            print(\"error\")\n",
    "                            print(refer_sentences_file)\n",
    "                            print(file)    \n",
    "                        \n",
    "                        parser1 = etree.XMLParser(recover=True)\n",
    "                       \n",
    "                        #print(\"Test:\"+xmlCitText)\n",
    "                        try:\n",
    "                            xmlEntryCit = \"<root>\"+xmlCitText+\"</root>\"\n",
    "                          \n",
    "                            xmlCitTree = ET.fromstring(xmlEntryCit.encode('UTF-8', 'ignore'),parser1)\n",
    "                        except:\n",
    "                            print(\"Except:\"+xmlEntryCit)\n",
    "                            print(\"\\n\"+str(counter))\n",
    "                            raise\n",
    "                        \n",
    "                        citEntries = parseSentXMLtree(xmlCitTree, citOffsets)\n",
    "                                                \n",
    "                        for citEntry in citEntries:\n",
    "                            if(citEntry[0]!=None):\n",
    "                                for negSent in negSentences:\n",
    "                                    negSentEnt = negSent.split(\"\\t\")\n",
    "                                    pair_off = citEntry[0] + \"\\t\" + negSentEnt[0] + \"\\t\" + str(citEntry[1]) + \"\\t\" + str(citEntry[2]) + \"\\t\" + str(negSentEnt[1]) + \"\\t\" + str(negSentEnt[2]) + \"\\t\" + name_C + \"\\t\" \n",
    "                                    pair = citEntry[0] + \"\\t\" + negSentEnt[0] \n",
    "                                    paircount = paircount+1\n",
    "                                    wf1.write(pair_off + \"\\n\")\n",
    "                                    wf2.write(pair + \"\\n\")\n",
    "                        \n",
    "                        \n",
    "                        all_cit = \"\"\n",
    "                        all_sid = []\n",
    "                        all_ssid = []\n",
    "                        for citEntry in citEntries:\n",
    "                            if(citEntry[0]!=None):\n",
    "                                all_cit+=citEntry[0]+\" \"\n",
    "                                all_sid.append(citEntry[1])\n",
    "                                all_ssid.append(citEntry[1])\n",
    "                        all_cit = all_cit.strip()\n",
    "                        for negSent in negSentences:\n",
    "                            negSentEnt = negSent.split(\"\\t\")\n",
    "                            pair_off = all_cit + \"\\t\" + negSentEnt[0] + \"\\t\" + str(all_sid) + \"\\t\" + str(all_ssid) + \"\\t\" + str(negSentEnt[1]) + \"\\t\" + str(negSentEnt[2]) + \"\\t\" + name_C + \"\\t\" \n",
    "                            pair = all_cit + \"\\t\" + negSentEnt[0] \n",
    "                            wf3.write(pair_off + \"\\n\")\n",
    "                            wf4.write(pair + \"\\n\")\n",
    "                        citEntries.clear()\n",
    "                        negSentences.clear()\n",
    "\n",
    "print(paircount)         \n",
    "print(counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#process negative pairs for ScisummNet data (2019)\n",
    "#1000 docs\n",
    "#generate all possible pairs (exhaustive)\n",
    "\n",
    "year = \"2019\"\n",
    "initDir = \"/home/chryssa/Work/CL-SciSumm/scisumm-corpus-master/From-ScisummNet-\"+year+\"_N/\"\n",
    "annotationFile = \".ann\"\n",
    "p = re.compile('\\([a-zA-Z]\\)')\n",
    "re.sub(' +', ' ', senText)\n",
    "m = p.match(senText)\n",
    "\n",
    "counter = 0;\n",
    "paircount = 0;\n",
    "writeDir1 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_offsets_corr_exhaustive_neg-\"+year+\"/\"\n",
    "writeDir2 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_corr_exhaustive_neg-\"+year+\"/\"\n",
    "writeDir3 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_offsets_full_corr_exhaustive_neg-\"+year+\"/\"\n",
    "writeDir4 = \"CL-SciSumm/scisumm-corpus-master/data/Training_pairs_full_corr_exhaustive_neg-\"+year+\"/\"\n",
    "\n",
    "if not os.path.exists(writeDir1):\n",
    "    os.makedirs(writeDir1)\n",
    "if not os.path.exists(writeDir2):\n",
    "    os.makedirs(writeDir2)\n",
    "if not os.path.exists(writeDir3):\n",
    "    os.makedirs(writeDir3)\n",
    "if not os.path.exists(writeDir4):\n",
    "    os.makedirs(writeDir4)\n",
    "    \n",
    "for root, dirs, files in os.walk(initDir):\n",
    "    for file in files:    \n",
    "        if annotationFile in file:\n",
    "             with open (os.path.join(root, file), \"r\") as annFile:\n",
    "                counter+=1\n",
    "                wf1 = open(writeDir1+file.split(\".\")[0]+\"_pairs_off.tsv\", \"w\") \n",
    "                wf2 = open(writeDir2+file.split(\".\")[0]+\"_pairs.tsv\", \"w\")    \n",
    "                wf3 = open(writeDir3+file.split(\".\")[0]+\"_pairs_off.tsv\", \"w\") \n",
    "                wf4 = open(writeDir4+file.split(\".\")[0]+\"_pairs.tsv\", \"w\")    \n",
    "        \n",
    "                \n",
    "                for line in annFile:\n",
    "                    if(len(line)>1):\n",
    "                        xmlCitText = \"\"\n",
    "                        xmlRefText = \"\"\n",
    "                        \n",
    "                        cit_num = \"0\"\n",
    "                        fields  = line.split(\" | \")\n",
    "                        for field in fields:\n",
    "                            if field.startswith(\"Citance Number:\"):\n",
    "                                start = field.find(\":\")\n",
    "                                cit_num = field[start:].strip()\n",
    "                            elif(field.startswith(\"Reference Article:\")):\n",
    "                                start = field.find(\":\")\n",
    "                                if(\".\" in field):\n",
    "                                    end = field.find(\".\", start)\n",
    "                                    name_R = field[start+1:end].strip()\n",
    "                                else:\n",
    "                                     name_R = field[start+1:].strip()\n",
    "                            elif field.startswith(\"Citing Article:\"):\n",
    "                                start = field.find(\":\") \n",
    "                                if(\".\" in field):\n",
    "                                    end = field.find(\".\", start)\n",
    "                                    name_C = field[start+1:end].strip()\n",
    "                                else:\n",
    "                                     name_C = field[start+1:].strip()\n",
    "                            elif field.startswith(\"Citation Offset\"):\n",
    "                                start = field.find(\"\\[\") \n",
    "                                end = field.find(\"\\]\", start)\n",
    "                                citOffset = field[start+1:end]\n",
    "                                citOffsets = citOffset.split(\",\")\n",
    "                            elif field.startswith(\"Citation Text:\"):\n",
    "                                start = field.find(\":\") \n",
    "                                end = field.find(\">\", start)\n",
    "                                xmlCitText = field[start+1:].strip()\n",
    "                            elif field.startswith(\"Reference Offset\"):\n",
    "                                start = field.find(\"\\[\") \n",
    "                                end = field.find(\"\\]\", start)\n",
    "                                refOffset = field[start+1:end]\n",
    "                                refOffsets = refOffset.split(\",\")\n",
    "                            elif field.startswith(\"Reference Text:\"):\n",
    "                                start = field.find(\":\") \n",
    "                                xmlRefText = field[start+1:].strip()\n",
    "                        \n",
    "                        #print(name_C)\n",
    "                        \n",
    "                        refer_sentences_file = \"CL-SciSumm/scisumm-corpus-master/data/Reference_Sent-\"+year+\"/\"+name_R+\"_sentences.txt\"\n",
    "                   \n",
    "                        lines = []\n",
    "                        negSentences= []\n",
    "                        if(os.path.isfile(refer_sentences_file)):\n",
    "                            with open(refer_sentences_file) as f:\n",
    "                                lines = f.read().splitlines()\n",
    "                                print(refer_sentences_file)\n",
    "                            max = len(lines)-1\n",
    "                            min = 0\n",
    "                            neg = [];\n",
    "                            if max>0:\n",
    "                                for lineNum in range(1,len(lines)):\n",
    "                                    candidate = lines[lineNum]\n",
    "                                    #print(candidate.split(\"\\t\")[1])\n",
    "                                    #print(candidate.split(\"\\t\")[1])\n",
    "                                    valid = True\n",
    "                                    if len(candidate.split(\"\\t\"))>1:\n",
    "                                        offsets = candidate.split(\"\\t\")[1].strip(\"\\[\\]\").split(\",\")\n",
    "                                    else:\n",
    "                                        valid = False\n",
    "                                    if not lineNum in neg:\n",
    "                                        for offset in offsets:\n",
    "                                            if offset in refOffsets:\n",
    "                                                 valid = False\n",
    "                                        qual = False\n",
    "                                        if valid:\n",
    "                                            qual = checkQuality(candidate.split(\"\\t\")[0])\n",
    "                                        if valid and qual:\n",
    "                                            neg.append(lineNum)\n",
    "                                            negSentences.append(candidate)\n",
    "                                        \n",
    "                        else:\n",
    "                            print(\"issue\")\n",
    "                            print(refer_sentences_file)\n",
    "                            print(file)    \n",
    "                        \n",
    "                       \n",
    "                        parser1 = etree.XMLParser(recover=True)\n",
    "                      \n",
    "                        #print(\"Test:\"+xmlCitText)\n",
    "                        try:\n",
    "                            xmlEntryCit = \"<root>\"+xmlCitText.replace(\"sid = \",\"sid =\\\"0\\\"\")+\"</root>\"\n",
    "                          \n",
    "                            xmlCitTree = ET.fromstring(xmlEntryCit.encode('UTF-8', 'ignore'),parser1)\n",
    "                        except:\n",
    "                            print(\"Except:\"+xmlEntryCit)\n",
    "                            print(\"\\n\"+str(counter))\n",
    "                            raise\n",
    "                       \n",
    "                        \n",
    "                        \n",
    "                        citEntries = parseSentXMLtree(xmlCitTree, citOffsets)\n",
    "                                                \n",
    "                        for citEntry in citEntries:\n",
    "                            if(citEntry[0]!=None):\n",
    "                                for negSent in negSentences:\n",
    "                                    negSentEnt = negSent.split(\"\\t\")  \n",
    "                                    pair_off = citEntry[0] + \"\\t\" + negSentEnt[0] + \"\\t\" + str(citEntry[1]) + \"\\t\" + str(citEntry[2]) + \"\\t\" + str(negSentEnt[1]) + \"\\t\" + str(negSentEnt[2]) + \"\\t\" + name_C\n",
    "                                    pair = citEntry[0] + \"\\t\" + negSentEnt[0] \n",
    "                                    paircount = paircount+1\n",
    "                                    wf1.write(pair_off + \"\\n\")\n",
    "                                    wf2.write(pair + \"\\n\")\n",
    "                            else:\n",
    "                                print(citEntry)\n",
    "                        \n",
    "                        all_cit = \"\"\n",
    "                        all_sid = []\n",
    "                        all_ssid = []\n",
    "                        if len(citEntries)>1:\n",
    "                            print(citEntries)\n",
    "                            for citEntry in citEntries:\n",
    "                                all_cit+=citEntry[0]+\" \"\n",
    "                                all_sid.append(citEntry[1])\n",
    "                                all_ssid.append(citEntry[1])\n",
    "                            all_cit = all_cit.strip()\n",
    "                            for negSent in negSentences:\n",
    "                                negSentEnt = negSent.split(\"\\t\")\n",
    "                                pair_off = all_cit + \"\\t\" + negSentEnt[0] + \"\\t\" + str(all_sid) + \"\\t\" + str(all_ssid) + \"\\t\" + str(negSentEnt[1]) + \"\\t\" + str(negSentEnt[2]) + \"\\t\" + name_C\n",
    "                                pair = all_cit + \"\\t\" + negSentEnt[0] \n",
    "                                wf3.write(pair_off + \"\\n\")\n",
    "                                wf4.write(pair + \"\\n\")\n",
    "                            citEntries.clear()\n",
    "                        negSentences.clear()\n",
    "\n",
    "print(paircount)         \n",
    "print(counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Process test set\n",
    "import csv\n",
    "initDir = \"CL-SciSumm/scisumm-corpus-master/data/Test-Set-2018/\"\n",
    "testExt = \".csv\"\n",
    "refExt = \"\"\n",
    "year=\"2018\"\n",
    "writeDir1 = \"CL-SciSumm/scisumm-corpus-master/data/Testing_pairs_offsets-\"+year+\"/\"\n",
    "writeDir2 = \"CL-SciSumm/scisumm-corpus-master/data/Testing_pairs-\"+year+\"/\"\n",
    "\n",
    "if not os.path.exists(writeDir1):\n",
    "    os.makedirs(writeDir1)\n",
    "if not os.path.exists(writeDir2):\n",
    "    os.makedirs(writeDir2)\n",
    "\n",
    "    \n",
    "\n",
    "#parse test csv\n",
    "for root, dirs, files in os.walk(initDir):\n",
    "    for file in files:    \n",
    "         if testExt in file:\n",
    "            with open (os.path.join(root, file), \"r\") as testFile:          \n",
    "                ref_name = file.split(\".\")[0]\n",
    "                ref_file = \"CL-SciSumm/scisumm-corpus-master/data/Test-Set-\"+year+\"/\"+ref_name+\"/Reference_XML/\"+ref_name+\".xml\"\n",
    "                sentences = []\n",
    "                #parse reference xml\n",
    "                with open(ref_file, encoding='utf-8', errors='ignore') as xml2str:\n",
    "                    data=xml2str.read()\n",
    "                    parser = ET.XMLParser()\n",
    "                    xmlTree = ET.fromstring(data,parser)\n",
    "\n",
    "                    for sent in xmlTree.iter(\"S\"):\n",
    "                        sid = sent.attrib['sid']\n",
    "                        if sid == \"0\" or sid ==\"\":\n",
    "                            ssid = \"0\"\n",
    "                        else:\n",
    "                            ssid = sent.attrib['ssid']\n",
    "                        senText = sent.text\n",
    "                        sentVec = [sid,ssid,senText]\n",
    "                        \n",
    "                        sentences.append(sentVec)\n",
    "                \n",
    "                wf1 = open(writeDir1+ref_name+\"_pairs_off.tsv\", \"w\") \n",
    "                wf2 = open(writeDir2+ref_name+\"_pairs.tsv\", \"w\") \n",
    "                #parse csv test file\n",
    "                csv_reader = csv.reader(testFile, delimiter=',')\n",
    "                line_count = 0\n",
    "                for row in csv_reader:\n",
    "                    if line_count == 0:\n",
    "                        print(f'Column names are {\", \".join(row)}')\n",
    "                        line_count += 1\n",
    "                    else:\n",
    "                        print(len(row))\n",
    "                        cit_num = row[0]\n",
    "                        ref_art = row[1]\n",
    "                        cit_art = row[2]\n",
    "                        cit_off = row[5]\n",
    "                        cit_text = row[6]\n",
    "                        cit_text_c = row[7]\n",
    "                        for sentV in sentences:\n",
    "                            pair = cit_text_c + \"\\t\" + sentV[2]\n",
    "                            pair_off = cit_text_c + \"\\t\" + sentV[2] + \"\\t\" + cit_art + \"\\t\" + cit_off + \"\\t\" + sentV[0] + \"\\t\" + sentV[1] + \"\\t\" + cit_art + \"\\t\" + cit_num\n",
    "                            print(pair)\n",
    "                            wf2.write(pair + \"\\n\")\n",
    "                            wf1.write(pair + \"\")\n",
    "                    \n",
    "                   \n",
    "                \n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Aug 2019\n",
    "#modifying data to use with fine-tune approach\n",
    "input_negative_pairs = \"NN_Training_pairs_offsets_full_corr_neg-2019/\"\n",
    "output_dir= \"20019-FT\"\n",
    "#outputDir2= \"/home/chryssa/Work/CL-SciSumm/scisumm-corpus-master/data/toMove/2018/train/fullCitance/sent2vec/references/\"\n",
    "\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)\n",
    "    \n",
    "trainLines=[]\n",
    "counter=0\n",
    "\n",
    "\n",
    "wf = open(os.path.join(output_dir,\"2019_cl_corpus_negative_pairs\" \"w\"))\n",
    "for root, dirs, files in os.walk(input_negative_pairs):\n",
    "    for file in files:    \n",
    "        with open (os.path.join(root, file), \"r\") as annFile:\n",
    "            for line in annFile:\n",
    "                fields = line.split(\"\\t\")\n",
    "                wf.write(fields[0])\n",
    "                wf.write(\"\\n\")\n",
    "                wf.write(fields[1])\n",
    "                wf.write(\"\\n\")\n",
    "                wf.write(\"\\n\")\n",
    "wf.close()"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}