SemEval2020_Task5_Data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SemEval2020_Task5_Data.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "lMnqA1JbeTGQ"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "\n",
        "!cp /content/gdrive/My\\ Drive/Colab\\ Notebooks/Subtask-2/test.csv /content\n",
        "!cp /content/gdrive/My\\ Drive/Colab\\ Notebooks/Subtask-2/train.csv /content\n",
        "\n",
        "!cp /content/gdrive/My\\ Drive/Colab\\ Notebooks/Subtask-2/antecedents.tsv /content\n",
        "!cp /content/gdrive/My\\ Drive/Colab\\ Notebooks/Subtask-2/consequents.tsv /content\n",
        "\n",
        "!cp /content/gdrive/My\\ Drive/Colab\\ Notebooks/Subtask-2/outputs.antecedents/submit_results.csv /content/antecedent_results.csv\n",
        "!cp /content/gdrive/My\\ Drive/Colab\\ Notebooks/Subtask-2/outputs.consequents/submit_results.csv /content/consequent_results.csv"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RsHDDbRWsPd-"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "prefix = '/content/'\n",
        "test_df = pd.read_csv(prefix + 'test.csv', header=None) # change to test.csv\n",
        "test_df=test_df.drop(index=0)\n",
        "\n",
        "test_df = pd.DataFrame({\n",
        "    'id':test_df[0],\n",
        "    'text': test_df[1]\n",
        "})\n",
        "# display \n",
        "test_df.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dbBm343LWeyp"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "prefix = '/content/'\n",
        "\n",
        "a_df = pd.read_csv(prefix + 'antecedents.tsv', sep='\\t', delimiter=None, header=None)\n",
        "a_df = pd.DataFrame({\n",
        "    'id': a_df[0],\n",
        "    'labels':a_df[1],\n",
        "    'text': a_df[2],\n",
        "    'start':a_df[3],\n",
        "    'end': a_df[4]\n",
        "})\n",
        "#a_df.sort_values(by=['id','text'], ascending=[1,0])\n",
        "\n",
        "c_df = pd.read_csv(prefix + 'consequents.tsv', sep='\\t', delimiter=None, header=None)\n",
        "c_df = pd.DataFrame({\n",
        "    'id': c_df[0],\n",
        "    'labels':c_df[1],\n",
        "    'text': c_df[2],\n",
        "    'start':c_df[3],\n",
        "    'end': c_df[4]\n",
        "})\n",
        "\n",
        "rc_df = pd.read_csv(prefix + 'consequent_results.csv', header=None)\n",
        "rc_df = pd.DataFrame({\n",
        "    'id': rc_df[0],\n",
        "    'labels':rc_df[1]\n",
        "})\n",
        "rc_df.sort_values(by=['labels'], ascending=False, inplace = True)\n",
        "\n",
        "ra_df = pd.read_csv(prefix + 'antecedent_results.csv', header=None)\n",
        "ra_df = pd.DataFrame({\n",
        "    'id': ra_df[0],\n",
        "    'labels':ra_df[1]\n",
        "})\n",
        "ra_df.sort_values(by=['labels'], ascending=False, inplace = True)\n",
        "ra_df.head(10)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PO8X5DmJxNUU"
      },
      "source": [
        "pip install allennlp allennlp-models"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MAzLpie_9b9w"
      },
      "source": [
        "from allennlp.predictors.predictor import Predictor\n",
        "import allennlp_models.structured_prediction\n",
        "predictor = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/biaffine-dependency-parser-ptb-2020.04.06.tar.gz\")\n",
        "\n",
        "original = 'If that was my daughter, I would have asked If I did something wrong.'    \n",
        "prediction = predictor.predict(\n",
        "  sentence = original\n",
        ")\n",
        "#prediction"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ze1FUy-yukEa"
      },
      "source": [
        "def extract_dependents(prediction):\n",
        " root = None\n",
        " text = str()\n",
        "\n",
        " for k, v in prediction.items():\n",
        "  if (k=='hierplane_tree'):\n",
        "    for key, val in v.items():\n",
        "      if key=='root':\n",
        "        root = val\n",
        "      if key=='text': \n",
        "        text = val   \n",
        "\n",
        " root.update({'text': text})\n",
        " return extract_children(root,'','advmod advcl nsubj ccomp xcomp pcomp pobj dobj iobj') "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Rrhac-EvZsE"
      },
      "source": [
        "def extract_children(obj, include='', exclude=''):\n",
        "  arr = list()\n",
        "  args = list()\n",
        "  spans = list()\n",
        "  frames = list()\n",
        "\n",
        "  def extract(obj, arr): \n",
        "    explore = False\n",
        "    limit = 5 \n",
        "    count = 0\n",
        "    text = str()\n",
        "\n",
        "    if isinstance(obj, dict):\n",
        "      for k, v in obj.items():\n",
        "        if include =='' and k=='attributes' and 'VERB' in v:\n",
        "          if obj not in frames:\n",
        "            frames.append(obj)\n",
        "\n",
        "        if k=='nodeType':\n",
        "          arr.append(v)\n",
        "\n",
        "        if k=='text' and len(v)>0 and len(text)==0:  \n",
        "          text = v\n",
        "\n",
        "        if k=='children' and isinstance(v, list):\n",
        "          if count < limit:\n",
        "            count = count + 1\n",
        "            extract(v, arr)   \n",
        "             \n",
        "    elif isinstance(obj, list):\n",
        "      for item in obj:\n",
        "        if isinstance(item, dict):\n",
        "    \n",
        "          for key, value in item.items():\n",
        "            if key == 'nodeType':\n",
        "              arr.append(value)\n",
        "              if (include=='' or value in include) and (exclude=='' or not [v for v in arr if v in exclude]):\n",
        "                #print('['+include+']', exclude,'=>', item['word'],'<=',arr)\n",
        "                if item not in args: \n",
        "                  args.append(item) # frame argument must be found amoung children objects\n",
        "                explore = True \n",
        "              else:  \n",
        "                explore = False\n",
        "\n",
        "            if key=='attributes' and 'CCONJ' in value:\n",
        "              explore = False     \n",
        "\n",
        "            if explore and key =='spans':\n",
        "              if value not in spans:\n",
        "                spans.append(value)  \n",
        "            \n",
        "          if explore and count < limit:\n",
        "            count = count + 1\n",
        "            extract(item, arr)\n",
        "\n",
        "        arr.clear() \n",
        "    return text\n",
        "\n",
        "  text = extract(obj, arr)\n",
        "  return text, spans, args, frames"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_J6ueg9eJ4RB"
      },
      "source": [
        "def position(span):\n",
        "  return span[0]['start']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c45HuHKFl-Ax"
      },
      "source": [
        "import re\n",
        "regex = '\\\\\\'\\s+([^\\']+)\\s+\\\\\\''\n",
        "def compact(fragment):\n",
        "# specify the number of replacements by changing the 4th argument\n",
        "  result = re.sub(regex, '\\''+r'\\1'+'\\'' , fragment)\n",
        "  return result"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4CoPtjXKBtrj"
      },
      "source": [
        "def normalize(fragment):\n",
        "  r = fragment.strip()\n",
        "  r = compact(r)\n",
        "  r = r.replace('   ',' ') \n",
        "  r = r.replace('  ',' ')\n",
        "  r = r.replace(' ,',',') \n",
        "  r = r.replace(' .','.')\n",
        "  r = r.replace('Mr.','Mr. ')\n",
        "  r = r.replace('Ms.','Ms. ')\n",
        "  r = r.replace('Mrs.','Mrs. ')\n",
        "  r = r.replace('.  ','. ') # Mr./Ms./Mrs.\n",
        "  r = r.replace('... ','...')\n",
        "  r = r.replace(' --- ','---')\n",
        "  r = r.replace(' -- ','--')\n",
        "  r = r.replace(' - ','-')\n",
        "  r = r.replace(' !','!')\n",
        "  r = r.replace(' ?','?')\n",
        "  r = r.replace('$ ','$')\n",
        "  r = r.replace('# ','#')\n",
        "  r = r.replace(' %','%')\n",
        "  r = r.replace('( ','(')\n",
        "  r = r.replace(' )',')')\n",
        "  r = r.replace(' ;',';')\n",
        "  r = r.replace(' :',':') \n",
        "  r = r.replace(': ',':') #50/50 - carefull!\n",
        "  r = r.replace(' nt','nt')\n",
        "  r = r.replace(' (k)','(k)') #401(k)\n",
        "  r = r.replace(' (tm)','(tm)')\n",
        "  r = r.replace('n na ','nna ')\n",
        "  r = r.replace(' \\'m ','\\'m ')\n",
        "  r = r.replace('s \\' ','s\\' ') # was 'neurtralized'?\n",
        "  r = r.replace(' \\'s ','\\'s ') # a 'surgical' strike!\n",
        "  r = r.replace(' \\'d ','\\'d ')\n",
        "  r = r.replace(' \\'re ','\\'re ')\n",
        "  r = r.replace(' \\'ve ','\\'ve ')\n",
        "  r = r.replace(' \\'ll ','\\'ll ')\n",
        "  r = r.replace(' n\\'t','n\\'t') # what about UPPER()?\n",
        "  r = r.replace(' \\' ',' \\'')\n",
        "  r = r.rstrip()\n",
        "  return r"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KAy762nn4sJX"
      },
      "source": [
        "def explore(frame, condition, complement, sentence, removed, include='',exclude='', astart=-1, aend=-1, cstart=-1, cend=-1):\n",
        "  deps = list()\n",
        "  loops= list()\n",
        "  if len(condition)==0:\n",
        "    return deps, loops\n",
        "\n",
        "  start = min(astart,cstart)\n",
        "  end = max(aend,cend)\n",
        "  spantext = sentence[start:end] # full (a-c) span(text can be 1 char off b/c sentence == original - \" \" removed )\n",
        "  if len(spantext) <= 1:\n",
        "    return deps, loops\n",
        "\n",
        "  if spantext[-1] in ',;:.?!': # remove any punctuation that could trigger the <if not> logic below\n",
        "    spantext = spantext[:-1]\n",
        " \n",
        "  if astart > cend: \n",
        "    if condition.split(' ')[0] in 'had should': # 'higher than the 50% that would have been expected |*had* the technique not been used'\n",
        "      spantext = str()\n",
        "    if complement.split(' ')[-1] in 'although': # 'I should have tried to talk him out of it as *though* | my life *had* depended on it.'\n",
        "      spantext = sentence[cstart:cend-10]       # 10 == len('as though')\n",
        "  elif cstart > aend: \n",
        "    if condition.split(' ')[-1] in 'although':  # since *though* in 'although', above ends^____________ will *not* loops.append(spantext)\n",
        "      spantext = sentence[astart:aend-10] \n",
        "    if complement.split(' ')[0] in 'had should':# 'Had he refused... , | he would have *had* to settle...' - will not unify (unfortunately!)\n",
        "      spantext = str() \n",
        "\n",
        "  if len(spantext) > 0:\n",
        "    if not (' even ' in spantext.lower() or # '...as such they could have trouble | buying *even* with a ridiculous amount of mone'- don't unify\n",
        "            'if ' in spantext.lower() or   # 'If that was my daughter | would have asked *If* I did something wrong' - will not unify (sadly)\n",
        "            #',' in spantext.lower() or\n",
        "            ';' in spantext.lower() or\n",
        "            ':' in spantext.lower() or \n",
        "            #'.' in spantext.lower() or     # Mr.\n",
        "            '?' in spantext.lower() or \n",
        "            '!' in spantext.lower()\n",
        "      ):\n",
        "      #print(start,end,spantext)\n",
        "      if ((0 < cstart and cstart <= aend + 2 and astart < cstart) or \n",
        "          (0 < cstart and astart <= cend + 2 and cstart < astart) or        # 'they had already said *what* | I would have said' - will unify\n",
        "          (0 < cstart and cstart == aend + 2 and condition.split(' ')[-1]  in 'although instead of i you we she he they that this these those which what whom who\\'s when and why') or \n",
        "          (0 < cstart and astart == cend + 2 and complement.split(' ')[-1] in 'although instead of i you we she he they that this these those which what whom who\\'s when and why')\n",
        "      ):\n",
        "        if spantext not in loops: # adjunct subspan\n",
        "          #print(spantext, cstart, cend, complement)\n",
        "          loops.append(spantext)\n",
        "\n",
        "  text, spans, args, frames = extract_children(frame, include, exclude)\n",
        "  if len(spans) == 0:\n",
        "    return deps, loops\n",
        "\n",
        "  for arg in args:    \n",
        "    #print('?\\\"'+arg['word']+'\\\":',astart,aend,condition) \n",
        "    for span in sorted(spans,key=position): # all spans\n",
        "      start=span[0]['start']\n",
        "      end = span[0]['end']\n",
        "      for r in removed:\n",
        "        if r[0] <= start:\n",
        "          start=start - 1\n",
        "        if r[0] <= end:\n",
        "          end = end - 1 \n",
        "      \n",
        "      spantext = sentence[start:end].strip() # dependent subspan of (a-)condition\n",
        "      #print(spantext,start,end)\n",
        "      if len(spantext) > 0 and len(spantext.split(' ')) == 1: # only a single word\n",
        "        if astart <= start and end <= aend: \n",
        "          if spantext in condition and spantext not in deps:\n",
        "            #print(spantext, start, end, condition)\n",
        "            deps.append(spantext)     \n",
        "            \n",
        "    return deps, loops       "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Bmabbq-MUbB"
      },
      "source": [
        "def track(frames, antecedent, consequent, sentence, removed, astart, aend, cstart, cend):\n",
        "  #print(astart, aend, cstart, cend)\n",
        "  adeps = set()\n",
        "  aloops= set()\n",
        "  cdeps = set()\n",
        "  cloops= set()\n",
        "  if len(frames)==0:\n",
        "    print('no frames detected!')\n",
        "    return adeps, cdeps, aloops, cloops\n",
        "\n",
        "  for frame in frames: \n",
        "    #print('frame:', frame['word'])\n",
        "    deps, loops = explore(frame, antecedent, consequent, sentence, removed, 'dep prep auxpass','advmod advcl nsubj ccomp xcomp pcomp pobj dobj iobj', astart, aend, cstart, cend)\n",
        "    adeps.update(deps)\n",
        "    aloops.update(loops)\n",
        "    deps, loops = explore(frame, consequent, antecedent, sentence, removed, 'advcl','',cstart,cend,astart,aend)\n",
        "    cdeps.update(deps)\n",
        "    cloops.update(loops)\n",
        "    if len(adeps) > 0 or len(cdeps) > 0:\n",
        "      return adeps, cdeps, aloops, cloops    \n",
        "    adeps = set()\n",
        "    cdeps = set()  \n",
        "\n",
        "  return adeps, cdeps, aloops, cloops   "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UBfvzUf-StyO"
      },
      "source": [
        "def separate_from(md, pa):\n",
        "  index = -1\n",
        "  if pa in md and len(md) > len(pa):\n",
        "    index = md.find(pa)\n",
        "    if index > 0 :\n",
        "      md = md[0:index]\n",
        "      #print('|'+md+'{...}')\n",
        "    elif index == 0:\n",
        "      md = md.replace(pa,'')\n",
        "      #print('|{...}'+md)  \n",
        "  return md, index       "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "msFC5TstNCsu"
      },
      "source": [
        "def record(original):\n",
        "  base=[(i, c) for i, c in enumerate(original)]\n",
        "  return base"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I-pu1f8IN7VA"
      },
      "source": [
        "def contract(text, base):\n",
        "  removed = []\n",
        "  for i in range(len(base)-1):\n",
        "    sequence = base[i][1]\n",
        "    if i < len(base) - 1 :\n",
        "      sequence = sequence + base[i+1][1]\n",
        "    if text.find(sequence,base[i][0]-i)==-1:\n",
        "      removed.append(base[i])\n",
        "  return removed"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "id": "RCRAUxWcq7Gp"
      },
      "source": [
        "original = 'If that was my daughter, I would have asked If I did something wrong.'\n",
        "\n",
        "prediction = predictor.predict(sentence = original.replace('\\\"',''))\n",
        "text, spans, args, frames = extract_dependents(prediction)\n",
        "#print(frames)\n",
        "print(text)\n",
        "\n",
        "antecedent = 'If I did something wrong'\n",
        "astart=44  \n",
        "aend = 67\n",
        "consequent = 'would have asked' \n",
        "cstart=27     \n",
        "cend = 42\n",
        "\n",
        "base = record(text)\n",
        "sentence = normalize(text)\n",
        "print(sentence)\n",
        "removed = contract(sentence, base)\n",
        "#print(removed)\n",
        "\n",
        "adeps, cdeps, aloops, cloops = track(frames, antecedent, consequent, original, removed, astart, aend, cstart, cend)\n",
        "print(adeps,cdeps)\n",
        "print(aloops,cloops)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vDSx2bPhYBFK"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "def analyze(id, df, r_df, original, sentence, frames=[], cprob=100, cstart=-1, cend=-1, first=True, override=False):\n",
        "  print('________________________________________________________________________________________________________________________')\n",
        "  apred = r_df.loc[r_df.id.str.contains(id)]\n",
        "  if len(apred)==0:\n",
        "    return id, 0, -1, -1, 0\n",
        "\n",
        "  ctext = original[cstart:cend+1] # text playing consequent role\n",
        "  print('|' if first else '|*','Fix','^=' if override else '=',ctext, cstart, cend)\n",
        " \n",
        "  # check based on rules first = rlab \n",
        "  alab ='0' # assume antecedent false\n",
        "  keep = False # dont keep if alab=0 \n",
        "\n",
        "  count= 0  # counter of predictions\n",
        "  for p in apred.index: # [p] = index\n",
        "    count = count + 1\n",
        "    amid = apred['id'][p]  \n",
        "    alab = apred['labels'][p] #predicted\n",
        "\n",
        "    arule = df.loc[df.id.str.contains(amid)]\n",
        "    assert len(arule) > 0\n",
        "\n",
        "    for a in arule.index: # [a] = index \n",
        "      ralab = arule['labels'][a] \n",
        "      astart = arule['start'][a]\n",
        "      aend = arule['end'][a]\n",
        "      #atext = arule['text'][a]\n",
        "\n",
        "      if ralab == 1:\n",
        "        keep = True # keep this in case of return\n",
        "        #if astart < cend and cend < aend: #  if any overlap present\n",
        "        #  return amid, alab, astart, aend, 0\n",
        "        #if cstart < aend and aend < cend: #    ^ aend|   \n",
        "          #if cstart > astart: # |        #   /      |          \n",
        "          #  aend = cstart - 1 # |astart aend+1=|cstart  cend| \n",
        "          #return amid, alab, astart, cend, 0 # splice astart with cend argument\n",
        "\n",
        "      atext = original[astart:aend+1]\n",
        "      print('|', ralab,'(',alab,')', atext, astart, aend) # atext is antecedent\n",
        "\n",
        "      if first and override:\n",
        "        return amid, 0, cstart, cend, 0 # pass through and return (c-)argument!!\n",
        "\n",
        "      if cprob == 50 and ( # span over previosly tried unsure(50) (a-c)arguments  \n",
        "          (cstart <= astart and aend <= cend) or\n",
        "          (astart <= cstart and cend <= aend)\n",
        "      ):\n",
        "        if (alab == 0 or not first) and len(apred.index) > count and not override:\n",
        "          print('x')\n",
        "          continue  \n",
        "        return amid, alab, min(astart,cstart), max(aend,cend) ,'-25' # very risky! \n",
        "\n",
        "      if ralab == 0:       \n",
        "        adeps,cdeps,aloops,cloops = track(frames, atext, ctext, sentence, removed, astart, aend, cstart, cend)\n",
        "        print('| Res =',adeps,cdeps,aloops,cloops) # eg. 'set() set() set() set()'\n",
        "\n",
        "        if alab == 0 and not keep: # alrady passed loop were alab==1\n",
        "          if len(aloops) == 0 and len(cloops) == 0 and cprob != '-25':\n",
        "            if atext not in ctext and ctext not in atext:\n",
        "              if cstart == aend + 2 and 'if ' not in ctext.lower(): # add more excluding conditions \n",
        "                aend = cend\n",
        "                print('|   +=', sentence[astart:aend+1])\n",
        "                return amid, alab, astart, aend,'-50'\n",
        "              elif astart == cend+2 and 'if ' not in atext.lower(): # add more excluding conditions\n",
        "                astart = cstart\n",
        "                print('|   +=', sentence[astart:aend+1])\n",
        "                return amid, alab, astart, aend,'-50'   \n",
        "        else: keep = True # keep this in case of return\n",
        "\n",
        "        if len(aloops)==1 or len(cloops)==1:\n",
        "          if not first and len(apred.index) > count and not override:\n",
        "            print('x')\n",
        "            continue\n",
        "          #if keep:\n",
        "          #  return amid, alab, astart, aend, 25 # deffer to classfier    \n",
        "          return amid, alab, min(astart,cstart), max(aend,cend) ,'-25'\n",
        "        elif len(adeps) > 0 and len(cdeps) > 0: \n",
        "          if not first and len(apred.index) > count and not override:\n",
        "            print('x')\n",
        "            continue \n",
        "          if keep:\n",
        "            return amid, alab, astart, aend, 25                \n",
        "          return amid, alab, astart, aend,'-100' # 100% unaceptable \n",
        "        elif len(adeps) > 0 or len(cdeps) > 0:\n",
        "          if not first and len(apred.index) > count and not override:\n",
        "            print('x')\n",
        "            continue \n",
        "          if keep:\n",
        "            return amid, alab, astart, aend, 25       \n",
        "          return amid, alab, astart, aend,'-75'   # 75% suspectable\n",
        "        else: # if not keep:  \n",
        "          if not first and len(apred.index) > count and not override:\n",
        "            print('x')\n",
        "            continue   \n",
        "                \n",
        "      if ralab== 1:\n",
        "        return amid, alab, astart, aend, 100 # nothing needs to be done\n",
        "      elif alab==1:\n",
        "        return amid, alab, astart, aend, 75 # can take a chance and see\n",
        "      elif keep:\n",
        "        return amid, alab, astart, aend, 25 # something stil can be done  \n",
        "      else:  \n",
        "        return amid, alab, astart, aend, 50 # nothing more can be done\n",
        "\n",
        "        \n",
        "      print('v')\n",
        "  return amid, alab, astart, aend, 0        # nothing was done"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WYK7SfuaSXOE"
      },
      "source": [
        "def recomb(a_df, ra_df, c_df, rc_df, para=[0,'empty']):\n",
        "  id = str(para[0])\n",
        "  original=para[1]\n",
        "\n",
        "  prediction = predictor.predict(sentence = original.replace('\\\"',''))\n",
        "\n",
        "  text, spans, args, frames = extract_dependents(prediction) # not passed as argument above\n",
        "  base = record(text)        # text re-generated from prediction includes extra spaces !!!\n",
        "  sentence = normalize(text) # remove all extra spaces \n",
        "  removed = contract(sentence, base) # keep removed char\n",
        "\n",
        "  print(para)\n",
        "\n",
        "  aprob = 100\n",
        "  astart=-1\n",
        "  aend = -1\n",
        "  cprob = 100\n",
        "  cstart=-1\n",
        "  cend = -1\n",
        " \n",
        "  cid, clab, cstart, cend, cprob = analyze(id, c_df, rc_df, original, sentence, frames, aprob, astart, aend)\n",
        "  print(cprob,'C--->',original[cstart:cend+1],cstart,cend)\n",
        "  aid, alab, astart, aend, aprob = analyze(id, a_df, ra_df, original, sentence, frames, cprob, cstart, cend, False)\n",
        "  print(aprob,'A--<-',original[astart:aend+1],astart, aend) \n",
        "\n",
        "  if aprob == '-25':\n",
        "    cid, clab, cstart, cend, cprob = analyze(id, c_df, rc_df, original, sentence, frames, aprob, astart, aend, False)\n",
        "    print(cprob,'C--->',original[cstart:cend+1],cstart,cend)\n",
        "    if cprob == 50:\n",
        "      return alab, astart, aend, aprob, clab, cstart, cend, cprob\n",
        "    \n",
        "  if cprob=='-25':\n",
        "    astart=cstart\n",
        "    aend = cend\n",
        "    cid, clab, cstart, cend, cprob = analyze(id, c_df, rc_df, original, sentence, frames, aprob, astart, aend)\n",
        "    print(cprob,'C--->',original[cstart:cend+1],cstart,cend) \n",
        "      \n",
        "  elif cprob=='-50':\n",
        "    astart=cstart\n",
        "    aend = cend        \n",
        "    cid, clab, cstart, cend, cprob = analyze(id, c_df, rc_df, original, sentence, frames, aprob, astart, aend,False)\n",
        "    print(cprob,'C--->',original[cstart:cend+1],cstart,cend)\n",
        "    if cprob=='-25':\n",
        "      cid, clab, cstart, cend, cprob = analyze(id, c_df, rc_df, original, sentence, frames, aprob, astart, aend,False, True)\n",
        "      print(cprob,'C--->',original[cstart:cend+1],cstart,cend)\n",
        "\n",
        "  elif cprob=='-75':\n",
        "    cid, clab, cstart, cend, cprob = analyze(id, c_df, rc_df, original, sentence, frames, aprob, astart, aend, False)\n",
        "    print(cprob,'C--->',original[cstart:cend+1],cstart,cend)\n",
        "    if cprob=='-25':\n",
        "      astart=cstart\n",
        "      aend = cend\n",
        "    if cprob=='-75' or cprob=='-25':  \n",
        "      cid, clab, cstart, cend, cprob = analyze(id, c_df, rc_df, original, sentence, frames, aprob, astart, aend,False, True)\n",
        "      print(cprob,'C--->',original[cstart:cend+1],cstart,cend)\n",
        "    \n",
        "  elif cprob=='-100':     \n",
        "    cid, clab, cstart, cend, cprob = analyze(id, c_df, rc_df, original, sentence, frames, aprob, astart, aend)\n",
        "    print(cprob,'C--->',original[cstart:cend+1],cstart,cend)\n",
        "   \n",
        "   \n",
        "  if aprob == '-25':\n",
        "    cstart=astart\n",
        "    cend = aend\n",
        "    aid, alab, astart, aend, aprob = analyze(id, a_df, ra_df, original, sentence, frames, cprob, cstart, cend)\n",
        "    print(aprob,'A--<-',original[astart:aend+1],astart,aend)\n",
        "    \n",
        "  elif aprob=='-50':\n",
        "    cstart=astart\n",
        "    cend = aend   \n",
        "    aid, alab, astart, aend, aprob = analyze(id, a_df, ra_df, original, sentence, frames, cprob, cstart, cend, False)\n",
        "    print(aprob,'A--<-',original[astart:aend+1],astart,aend)\n",
        "    if aprob=='-25':  \n",
        "      aid, alab, astart, aend, aprob = analyze(id, a_df, ra_df, original, sentence, frames, cprob, cstart, cend,False, True)\n",
        "      print(aprob,'A--<-',original[astart:aend+1],astart,aend)\n",
        "    \n",
        "  elif aprob=='-75':  \n",
        "    aid, alab, astart, aend, aprob = analyze(id, a_df, ra_df, original, sentence, frames, cprob, cstart, cend, False)\n",
        "    print(aprob,'A--<-',original[astart:aend+1],astart,aend)\n",
        "    if aprob=='-25':\n",
        "      cstart=astart\n",
        "      cend = aend\n",
        "    if aprob=='-75' or aprob=='-25':  \n",
        "      aid, alab, astart, aend, aprob = analyze(id, a_df, ra_df, original, sentence, frames, cprob, cstart, cend,False, True)\n",
        "      print(aprob,'A--<-',original[astart:aend+1],astart,aend)\n",
        "        \n",
        "\n",
        "  if aprob == 25:\n",
        "    aid, alab, astart, aend, aprob = analyze(id, a_df, ra_df, original, sentence, frames, cprob, cstart, cend, False, True)\n",
        "    print(aprob,'A--<-',original[astart:aend+1],astart,aend) \n",
        "\n",
        "  if cprob == 25:\n",
        "    cid, clab, cstart, cend, cprob = analyze(id, c_df, rc_df, original, sentence, frames, aprob, astart, aend, False, True)\n",
        "    print(cprob,'C--->',original[cstart:cend+1],cstart,cend)\n",
        "\n",
        "\n",
        "  return alab, astart, aend, aprob, clab, cstart, cend, cprob"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vTneeOFQT21B"
      },
      "source": [
        "para = ['205487', '\"\"It\\'s once more piece of information I could have used in making my decision, should I have had it available at the time\" she says.\"']\n",
        "\n",
        "\n",
        "original = para[1]\n",
        "alab, astart, aend, aprob, clab, cstart, cend, cprob = recomb(a_df, ra_df, c_df, rc_df, para)\n",
        "\n",
        "consequent = original[cstart:cend+1]\n",
        "antecedent = original[astart:aend+1]\n",
        "\n",
        "cdep = '' # Pass\n",
        "if consequent == antecedent:\n",
        "  consequent = ''\n",
        "  cstart=-1\n",
        "  cend = -1\n",
        "  cdep = '='\n",
        "\n",
        "if len(consequent) > 0:\n",
        "  if consequent in antecedent:\n",
        "    consequent = ''\n",
        "    cstart=-1\n",
        "    cend = -1\n",
        "    cdep = '<'\n",
        "  elif antecedent in consequent:\n",
        "    antecedent = consequent\n",
        "    astart=cstart\n",
        "    aend = cend\n",
        "    aprob= cprob\n",
        "    consequent = ''\n",
        "    cstart=-1\n",
        "    cend = -1\n",
        "    cdep = '>'\n",
        "  \n",
        "if len(antecedent) > 0:       \n",
        "  if len(consequent) > 0:\n",
        "    text = antecedent + ' | ' + consequent\n",
        "  else:\n",
        "    text = antecedent\n",
        "else:\n",
        "  cdep = '!'     \n",
        "\n",
        "print(text, astart, aend, cstart, cend)\n",
        "print(cdep,'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "both",
        "id": "QUPpOr6t6udV"
      },
      "source": [
        "from tqdm import tqdm\n",
        "from time import sleep\n",
        "\n",
        "submit_df = pd.DataFrame(columns=['sentenceID','antecedent_startid','antecedent_endid','consequent_startid','consequent_endid'])\n",
        "report_df = pd.DataFrame(columns=['sentenceID','clabel','cprob','cdep','consequent','cstart','cend','antecedent','astart','aend','alabel','aprob'])\n",
        "# remove all gold_ for test\n",
        "\n",
        "count = 0\n",
        "for ind in tqdm(test_df.index): \n",
        "  original = test_df['text'][ind]\n",
        "  id = test_df['id'][ind]\n",
        "\n",
        "  para = [id, original]\n",
        "  alab, astart, aend, aprob, clab, cstart, cend, cprob = recomb(a_df, ra_df, c_df, rc_df, para)\n",
        "\n",
        "  consequent = original[cstart:cend+1]\n",
        "  antecedent = original[astart:aend+1] \n",
        "\n",
        "  cdep = '' # Pass\n",
        "  if consequent == antecedent:\n",
        "    consequent = ''\n",
        "    cstart=-1\n",
        "    cend = -1\n",
        "    cdep = '='\n",
        "\n",
        "  if len(consequent) > 0:\n",
        "    if consequent in antecedent:\n",
        "      consequent = ''\n",
        "      cstart=-1\n",
        "      cend = -1\n",
        "      cdep = '<'\n",
        "    elif antecedent in consequent:\n",
        "      antecedent = consequent\n",
        "      astart=cstart\n",
        "      aend = cend\n",
        "      aprob= cprob\n",
        "      consequent = ''\n",
        "      cstart=-1\n",
        "      cend = -1\n",
        "      cdep = '>'\n",
        "    \n",
        "  if len(antecedent) > 0:       \n",
        "    if len(consequent) > 0:\n",
        "      text = antecedent + ' | ' + consequent\n",
        "    else:\n",
        "      text = antecedent\n",
        "  else:\n",
        "    cdep = '!'      \n",
        "\n",
        "  if len(cdep) > 0:\n",
        "    count=count+1\n",
        "    \n",
        "  report_row = pd.Series( data={\n",
        "    'sentenceID': id,\n",
        "    'clabel': clab,\n",
        "    'cprob': cprob,\n",
        "    'cdep' : cdep,\n",
        "    'consequent': consequent, # comment all gold_... \n",
        "    #'gold_consequent': test_df['consequent'][ind],\n",
        "    'cstart': cstart,  \n",
        "    #'gold_cstart': test_df['cstart'][ind],       \n",
        "    'cend': cend,\n",
        "    #'gold_cend': test_df['cend'][ind],\n",
        "    #'gold_antecedent': test_df['antecedent'][ind],\n",
        "    'antecedent': antecedent, \n",
        "    'astart': astart,\n",
        "    #'gold_astart': test_df['astart'][ind], \n",
        "    'aend': aend,\n",
        "    #'gold_aend': test_df['aend'][ind],\n",
        "    'alabel': alab,\n",
        "    'aprob': aprob\n",
        "  }, name = str(id) )\n",
        "\n",
        "  result_row = pd.Series( data={\n",
        "    'sentenceID': id,\n",
        "    'antecedent_startid':astart,\n",
        "    'antecedent_endid': aend,\n",
        "    'consequent_startid': cstart,\n",
        "    'consequent_endid': cend\n",
        "  }, name = str(id) )\n",
        "\n",
        "  submit_df = submit_df.append(result_row, ignore_index=True)\n",
        "  report_df = report_df.append(report_row, ignore_index=True)\n",
        "\n",
        "  print(text, astart, aend, cstart, cend)\n",
        "  print(cdep,'~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')\n",
        "  sleep(1)\n",
        "\n",
        "submit_df.to_csv(prefix+'subtask2.csv',index=False, header=['sentenceID','antecedent_startid','antecedent_endid','consequent_startid','consequent_endid']) \n",
        "report_df.to_csv(prefix+'report.csv',index=False, header=['sentenceID','clabel','cprob','cdep','consequent','cstart','cend','antecedent','astart','aend','alabel','aprob'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ib77s9dReMcD"
      },
      "source": [
        "!zip submission_task2.zip subtask2.csv\n",
        "!mv /content/submission_task2.zip /content/gdrive/My\\ Drive/Colab\\ Notebooks/Subtask-2/\n",
        "!mv /content/report.csv /content/gdrive/My\\ Drive/Colab\\ Notebooks/Subtask-2/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mAPEdzUiyfkz"
      },
      "source": [
        "import pytz\n",
        "from datetime import datetime, timezone\n",
        "EST = pytz.timezone('US/Eastern')\n",
        "print('Adjusted =',count,'on',datetime.now(EST))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}