dep_parse.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "dep_parse.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "gnS2sY9VL0rw",
        "colab_type": "code",
        "outputId": "87fe8c32-8549-4ca7-c4b1-ddf8e65090fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        }
      },
      "source": [
        "!pip install stanfordnlp\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: stanfordnlp in /usr/local/lib/python3.6/dist-packages (0.1.2)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (1.0.1.post2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (4.28.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (1.14.6)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (3.7.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (2.18.4)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->stanfordnlp) (40.9.0)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf->stanfordnlp) (1.11.0)\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (1.22)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (2019.3.9)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (2.6)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PkHGPP4vOBBY",
        "colab_type": "code",
        "outputId": "0bfcc628-ab44-49c9-da88-59fd133fff54",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313
        }
      },
      "source": [
        "!pip3 install stanfordnlp\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting stanfordnlp\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c6/53/86245cebb380fb5f7f5e16eccfe78afed8c3c2c7ef218331cbcafce2be18/stanfordnlp-0.1.2-py3-none-any.whl (135kB)\n",
            "\u001b[K    100% |████████████████████████████████| 143kB 13.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (1.14.6)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (4.28.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (2.18.4)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (3.7.1)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from stanfordnlp) (1.0.1.post2)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (3.0.4)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (2.6)\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (1.22)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->stanfordnlp) (2019.3.9)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf->stanfordnlp) (1.11.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->stanfordnlp) (40.9.0)\n",
            "Installing collected packages: stanfordnlp\n",
            "Successfully installed stanfordnlp-0.1.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hWoiZlpAOIZ8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import stanfordnlp"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ufyreiBYOMeb",
        "colab_type": "code",
        "outputId": "96d297b8-7814-4e0f-e531-62b10f654678",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        }
      },
      "source": [
        "stanfordnlp.download('en')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using the default treebank \"en_ewt\" for language \"en\".\n",
            "Would you like to download the models for: en_ewt now? (Y/n)\n",
            "y\n",
            "\n",
            "Default download directory: /root/stanfordnlp_resources\n",
            "Hit enter to continue or type an alternate directory.\n",
            "\n",
            "\n",
            "Downloading models for: en_ewt\n",
            "Download location: /root/stanfordnlp_resources/en_ewt_models.zip\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 1.96G/1.96G [03:53<00:00, 8.51MB/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Download complete.  Models saved to: /root/stanfordnlp_resources/en_ewt_models.zip\n",
            "Extracting models file for: en_ewt\n",
            "Cleaning up...Done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AFYmOV4vPEbh",
        "colab_type": "code",
        "outputId": "a9b7acea-8770-47f1-b2ee-147dd78c90fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 451
        }
      },
      "source": [
        "nlp = stanfordnlp.Pipeline(processors = \"tokenize,mwt,lemma,pos,depparse\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Use device: gpu\n",
            "---\n",
            "Loading: tokenize\n",
            "With settings: \n",
            "{'model_path': '/root/stanfordnlp_resources/en_ewt_models/en_ewt_tokenizer.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
            "---\n",
            "Loading: lemma\n",
            "With settings: \n",
            "{'model_path': '/root/stanfordnlp_resources/en_ewt_models/en_ewt_lemmatizer.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
            "Building an attentional Seq2Seq model...\n",
            "Using a Bi-LSTM encoder\n",
            "Using soft attention for LSTM.\n",
            "Finetune all embeddings.\n",
            "[Running seq2seq lemmatizer with edit classifier]\n",
            "---\n",
            "Loading: pos\n",
            "With settings: \n",
            "{'model_path': '/root/stanfordnlp_resources/en_ewt_models/en_ewt_tagger.pt', 'pretrain_path': '/root/stanfordnlp_resources/en_ewt_models/en_ewt.pretrain.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
            "---\n",
            "Loading: depparse\n",
            "With settings: \n",
            "{'model_path': '/root/stanfordnlp_resources/en_ewt_models/en_ewt_parser.pt', 'pretrain_path': '/root/stanfordnlp_resources/en_ewt_models/en_ewt.pretrain.pt', 'lang': 'en', 'shorthand': 'en_ewt', 'mode': 'predict'}\n",
            "Done loading processors!\n",
            "---\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9wsAW7ieNYBF",
        "colab_type": "code",
        "outputId": "93d20b91-4024-44b3-e250-7b63acd5f678",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        }
      },
      "source": [
        "doc = nlp(\"Mariel Zagunis is notable for winning what?\")\n",
        "doc.sentences[0].print_tokens()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<Token index=1;words=[<Word index=1;text=Mariel;lemma=Mariel;upos=PROPN;xpos=NNP;feats=Number=Sing;governor=4;dependency_relation=nsubj>]>\n",
            "<Token index=2;words=[<Word index=2;text=Zagunis;lemma=Zagunis;upos=PROPN;xpos=NNP;feats=Number=Sing;governor=1;dependency_relation=flat>]>\n",
            "<Token index=3;words=[<Word index=3;text=is;lemma=be;upos=AUX;xpos=VBZ;feats=Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin;governor=4;dependency_relation=cop>]>\n",
            "<Token index=4;words=[<Word index=4;text=notable;lemma=notable;upos=ADJ;xpos=JJ;feats=Degree=Pos;governor=0;dependency_relation=root>]>\n",
            "<Token index=5;words=[<Word index=5;text=for;lemma=for;upos=SCONJ;xpos=IN;feats=_;governor=6;dependency_relation=mark>]>\n",
            "<Token index=6;words=[<Word index=6;text=winning;lemma=win;upos=VERB;xpos=VBG;feats=VerbForm=Ger;governor=4;dependency_relation=advcl>]>\n",
            "<Token index=7;words=[<Word index=7;text=what;lemma=what;upos=PRON;xpos=WP;feats=PronType=Int;governor=6;dependency_relation=obj>]>\n",
            "<Token index=8;words=[<Word index=8;text=?;lemma=?;upos=PUNCT;xpos=.;feats=_;governor=4;dependency_relation=punct>]>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X56hwLgwRC2_",
        "colab_type": "code",
        "outputId": "e81a4fc0-4ce4-4e15-bc1c-7ab768aecf94",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 155
        }
      },
      "source": [
        "#doc.sentences[0].print_dependencies()\n",
        "\n",
        "dependencies = doc.sentences[0].dependencies\n",
        "\n",
        "for i in dependencies:\n",
        "  print(i[1])\n",
        "\n",
        "  \n",
        "\n",
        "#doc.sentences[0].print_tokens()\n",
        "#for i in range(doc.sentences[0].print_dependencies()):\n",
        "#  print(i[2])\n",
        "#print(my_list)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "nsubj\n",
            "flat\n",
            "cop\n",
            "root\n",
            "mark\n",
            "advcl\n",
            "obj\n",
            "punct\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A_Aqje5uV1wv",
        "colab_type": "code",
        "outputId": "3fe9bd95-8c84-4b00-b5d1-e031072bda2f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        }
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "#extract lemma\n",
        "def extract_lemma(doc):\n",
        "    parsed_text = {'word':[], 'lemma':[]}\n",
        "    for sent in doc.sentences:\n",
        "        for wrd in sent.words:\n",
        "            #extract text and lemma\n",
        "            parsed_text['word'].append(wrd.text)\n",
        "            parsed_text['lemma'].append(wrd.lemma)\n",
        "    #return a dataframe\n",
        "    return pd.DataFrame(parsed_text)\n",
        "\n",
        "#call the function on doc\n",
        "extract_lemma(doc)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>lemma</th>\n",
              "      <th>word</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Mariel</td>\n",
              "      <td>Mariel</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Zagunis</td>\n",
              "      <td>Zagunis</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>be</td>\n",
              "      <td>is</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>notable</td>\n",
              "      <td>notable</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>for</td>\n",
              "      <td>for</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>win</td>\n",
              "      <td>winning</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>what</td>\n",
              "      <td>what</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>?</td>\n",
              "      <td>?</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     lemma     word\n",
              "0   Mariel   Mariel\n",
              "1  Zagunis  Zagunis\n",
              "2       be       is\n",
              "3  notable  notable\n",
              "4      for      for\n",
              "5      win  winning\n",
              "6     what     what\n",
              "7        ?        ?"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T_3HwYkfVezW",
        "colab_type": "code",
        "outputId": "2202b99f-89fe-4af3-f487-f9ba9d00a0fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        }
      },
      "source": [
        "pos_dict = {\n",
        "'CC': 'coordinating conjunction','CD': 'cardinal digit','DT': 'determiner',\n",
        "'EX': 'existential there (like: \\\"there is\\\" ... think of it like \\\"there exists\\\")',\n",
        "'FW': 'foreign word','IN':  'preposition/subordinating conjunction','JJ': 'adjective \\'big\\'',\n",
        "'JJR': 'adjective, comparative \\'bigger\\'','JJS': 'adjective, superlative \\'biggest\\'',\n",
        "'LS': 'list marker 1)','MD': 'modal could, will','NN': 'noun, singular \\'desk\\'',\n",
        "'NNS': 'noun plural \\'desks\\'','NNP': 'proper noun, singular \\'Harrison\\'',\n",
        "'NNPS': 'proper noun, plural \\'Americans\\'','PDT': 'predeterminer \\'all the kids\\'',\n",
        "'POS': 'possessive ending parent\\'s','PRP': 'personal pronoun I, he, she',\n",
        "'PRP$': 'possessive pronoun my, his, hers','RB': 'adverb very, silently,',\n",
        "'RBR': 'adverb, comparative better','RBS': 'adverb, superlative best',\n",
        "'RP': 'particle give up','TO': 'to go \\'to\\' the store.','UH': 'interjection errrrrrrrm',\n",
        "'VB': 'verb, base form take','VBD': 'verb, past tense took',\n",
        "'VBG': 'verb, gerund/present participle taking','VBN': 'verb, past participle taken',\n",
        "'VBP': 'verb, sing. present, non-3d take','VBZ': 'verb, 3rd person sing. present takes',\n",
        "'WDT': 'wh-determiner which','WP': 'wh-pronoun who, what','WP$': 'possessive wh-pronoun whose',\n",
        "'WRB': 'wh-abverb where, when','QF' : 'quantifier, bahut, thoda, kam (Hindi)','VM' : 'main verb',\n",
        "'PSP' : 'postposition, common in indian langs','DEM' : 'demonstrative, common in indian langs'\n",
        "}\n",
        "\n",
        "\n",
        "#extract parts of speech\n",
        "parsed_text = {'word':[], 'pos':[], 'exp':[]}\n",
        "def extract_pos(doc):\n",
        "    for sent in doc.sentences:\n",
        "        for wrd in sent.words:\n",
        "            if wrd.pos in pos_dict.keys():\n",
        "                pos_exp = pos_dict[wrd.pos]\n",
        "            else:\n",
        "                pos_exp = 'NA'\n",
        "            parsed_text['word'].append(wrd.text)\n",
        "            parsed_text['pos'].append(wrd.pos)\n",
        "            parsed_text['exp'].append(pos_exp)\n",
        "    #return a dataframe of pos and text\n",
        "    return pd.DataFrame(parsed_text)\n",
        "\n",
        "#extract pos\n",
        "extract_pos(doc)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>exp</th>\n",
              "      <th>pos</th>\n",
              "      <th>word</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>proper noun, singular 'Harrison'</td>\n",
              "      <td>NNP</td>\n",
              "      <td>Mariel</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>proper noun, singular 'Harrison'</td>\n",
              "      <td>NNP</td>\n",
              "      <td>Zagunis</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>verb, 3rd person sing. present takes</td>\n",
              "      <td>VBZ</td>\n",
              "      <td>is</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>adjective 'big'</td>\n",
              "      <td>JJ</td>\n",
              "      <td>notable</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>preposition/subordinating conjunction</td>\n",
              "      <td>IN</td>\n",
              "      <td>for</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>verb, gerund/present participle taking</td>\n",
              "      <td>VBG</td>\n",
              "      <td>winning</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>wh-pronoun who, what</td>\n",
              "      <td>WP</td>\n",
              "      <td>what</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>NA</td>\n",
              "      <td>.</td>\n",
              "      <td>?</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      exp  pos     word\n",
              "0        proper noun, singular 'Harrison'  NNP   Mariel\n",
              "1        proper noun, singular 'Harrison'  NNP  Zagunis\n",
              "2    verb, 3rd person sing. present takes  VBZ       is\n",
              "3                         adjective 'big'   JJ  notable\n",
              "4   preposition/subordinating conjunction   IN      for\n",
              "5  verb, gerund/present participle taking  VBG  winning\n",
              "6                    wh-pronoun who, what   WP     what\n",
              "7                                      NA    .        ?"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3UvU00igoY1D",
        "colab_type": "code",
        "outputId": "a485d9aa-f04b-476b-aa78-34bbdf756acf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "print(parsed_text['pos'])\n",
        "print(parsed_text['word'])\n",
        "print(len(parsed_text['pos']))\n",
        "print(len(parsed_text['word']))\n",
        "print(\"----------------------------------------------------------\")\n",
        "#print(len(parsed_text['word']))\n",
        "\n",
        "dependencies = doc.sentences[0].dependencies\n",
        "\n",
        "LAT_Index = 0\n",
        "proceed = True \n",
        "QuestionWord_index=0\n",
        "if(len(parsed_text['pos']) == len(dependencies)):\n",
        "  count = 0\n",
        "  Question_word_string= \"\"\n",
        "\n",
        "  for pos in parsed_text['pos']:\n",
        "    if(pos==\"WDT\" or pos==\"WRB\" or pos==\"WP\"):\n",
        "      QuestionWord_index = count\n",
        "      Question_word_string = parsed_text['word'][QuestionWord_index]\n",
        "      if(Question_word_string.lower() == \"when\" or Question_word_string.lower() == \"where\" or Question_word_string.lower() == \"who\"):\n",
        "        proceed = False\n",
        "\n",
        "      break\n",
        "    count=count+1\n",
        "\n",
        "  question_word_pos_index = 0\n",
        "  window_size =0\n",
        "  count_1 = 0\n",
        "  dependencylist = []\n",
        "  #dependency_list['dependencylist'] = []\n",
        "\n",
        "  #window size logic\n",
        "  flag = False\n",
        "  for i in dependencies:\n",
        "    if(flag):\n",
        "      print(i[1])\n",
        "      dependency_list = i[1]\n",
        "\n",
        "      if(parsed_text['pos'][count_1] != \"NN\" ):\n",
        "        window_size = 5\n",
        "        print(\"window_size\")\n",
        "        print(window_size)\n",
        "      else:\n",
        "        window_size = 3\n",
        "        print(\"window_size\")\n",
        "        print(window_size)\n",
        "      break\n",
        "    if(count_1 == QuestionWord_index):\n",
        "      if(parsed_text['pos'][count_1] == \"WDT\" or parsed_text['pos'][count_1] == \"WRB\" or parsed_text['pos'][count_1] == \"WP\"):\n",
        "        question_word_pos_index = count_1    \n",
        "        flag = True\n",
        "      # write one more condition for corner case where i is in hr   \n",
        "    count_1 = count_1 +1\n",
        "\n",
        "  print(\"window size=\")\n",
        "  print(window_size)\n",
        "\n",
        "\n",
        "  for i in dependencies:\n",
        "    dependencylist.append(i[1])\n",
        "\n",
        "\n",
        "  subject_found = False\n",
        "  # window logic \n",
        "  # write corner\n",
        "\n",
        "  boundary = 0\n",
        "\n",
        "#  if(question_word_pos_index == len(parsed_text['word']) or question_word_pos_index +1 == len(parsed_text['word'])):\n",
        " #    print(\"corner case\")\n",
        "\n",
        "\n",
        "  if question_word_pos_index + window_size < len(parsed_text['word']):\n",
        "    boundary = question_word_pos_index + window_size + 1\n",
        "  else:\n",
        "    boundary = len(parsed_text['word'])\n",
        "\n",
        "  first_noun_found = False\n",
        "  #question_word_pos_index +1\n",
        "  LAT_found= False\n",
        "\n",
        "\n",
        "  if(Question_word_string.lower() == \"how\"):\n",
        "    if(QuestionWord_index+1 <= len(parsed_text['word'])-1):\n",
        "      if(parsed_text['pos'][QuestionWord_index+1] == \"JJ\"):\n",
        "        LAT_Index = QuestionWord_index+1  \n",
        "  #Question_word_string\n",
        "  else:\n",
        "    for index in range(question_word_pos_index + 1, boundary):\n",
        "      if(not first_noun_found):\n",
        "        if(parsed_text['pos'][index] == \"NN\" or parsed_text['pos'][index] == \"NNS\" or parsed_text['pos'][index] == \"NNP\"):\n",
        "          LAT_Index = index\n",
        "          first_noun_found = True\n",
        "      if(dependencylist[index] == \"nsubj\" or dependencylist[index] == \"nsubjpass\" or dependencylist[index] == \"nsubj:pass\"):   \n",
        "        subject_found = True\n",
        "        LAT_Index = index\n",
        "        break\n",
        "\n",
        "if(not proceed):\n",
        "  print(parsed_text['word'][QuestionWord_index])\n",
        "else:\n",
        "  print(parsed_text['word'][LAT_Index])\n",
        "\n",
        "#   if(dependencylist[index] == \"obj\"):\n",
        "#     if(subject_found == False): # has doubt with it\n",
        "#       LAT_Index = index\n",
        "\n",
        "# case equals in python  \n",
        "\n",
        "    \n",
        "      \n",
        "\n",
        " \n",
        "  "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['NNP', 'NNP', 'VBZ', 'JJ', 'IN', 'VBG', 'WP', '.']\n",
            "['Mariel', 'Zagunis', 'is', 'notable', 'for', 'winning', 'what', '?']\n",
            "8\n",
            "8\n",
            "----------------------------------------------------------\n",
            "punct\n",
            "window_size\n",
            "5\n",
            "window size=\n",
            "5\n",
            "Mariel\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9HnimIjK0nc8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_LAT(text, nlp):\n",
        "  doc = nlp(text)\n",
        "  pos_dict = {\n",
        "  'CC': 'coordinating conjunction','CD': 'cardinal digit','DT': 'determiner',\n",
        "  'EX': 'existential there (like: \\\"there is\\\" ... think of it like \\\"there exists\\\")',\n",
        "  'FW': 'foreign word','IN':  'preposition/subordinating conjunction','JJ': 'adjective \\'big\\'',\n",
        "  'JJR': 'adjective, comparative \\'bigger\\'','JJS': 'adjective, superlative \\'biggest\\'',\n",
        "  'LS': 'list marker 1)','MD': 'modal could, will','NN': 'noun, singular \\'desk\\'',\n",
        "  'NNS': 'noun plural \\'desks\\'','NNP': 'proper noun, singular \\'Harrison\\'',\n",
        "  'NNPS': 'proper noun, plural \\'Americans\\'','PDT': 'predeterminer \\'all the kids\\'',\n",
        "  'POS': 'possessive ending parent\\'s','PRP': 'personal pronoun I, he, she',\n",
        "  'PRP$': 'possessive pronoun my, his, hers','RB': 'adverb very, silently,',\n",
        "  'RBR': 'adverb, comparative better','RBS': 'adverb, superlative best',\n",
        "  'RP': 'particle give up','TO': 'to go \\'to\\' the store.','UH': 'interjection errrrrrrrm',\n",
        "  'VB': 'verb, base form take','VBD': 'verb, past tense took',\n",
        "  'VBG': 'verb, gerund/present participle taking','VBN': 'verb, past participle taken',\n",
        "  'VBP': 'verb, sing. present, non-3d take','VBZ': 'verb, 3rd person sing. present takes',\n",
        "  'WDT': 'wh-determiner which','WP': 'wh-pronoun who, what','WP$': 'possessive wh-pronoun whose',\n",
        "  'WRB': 'wh-abverb where, when','QF' : 'quantifier, bahut, thoda, kam (Hindi)','VM' : 'main verb',\n",
        "  'PSP' : 'postposition, common in indian langs','DEM' : 'demonstrative, common in indian langs'\n",
        "   }  \n",
        "\n",
        "  parsed_text = {'word':[], 'pos':[], 'exp':[]}\n",
        "  for sent in doc.sentences:\n",
        "    for wrd in sent.words:\n",
        "      if wrd.pos in pos_dict.keys():\n",
        "        pos_exp = pos_dict[wrd.pos]\n",
        "      else:\n",
        "        pos_exp = 'NA'\n",
        "      parsed_text['word'].append(wrd.text)\n",
        "      parsed_text['pos'].append(wrd.pos)\n",
        "      parsed_text['exp'].append(pos_exp)\n",
        " \n",
        "  print(parsed_text['pos'])\n",
        "  print(parsed_text['word'])\n",
        "  print(len(parsed_text['pos']))\n",
        "  print(len(parsed_text['word']))\n",
        "  \n",
        "  LAT_String = \"\"\n",
        "  dependencies = doc.sentences[0].dependencies\n",
        "\n",
        "  LAT_Index = 0\n",
        "  proceed = True \n",
        "  QuestionWord_index=0\n",
        "  if(len(parsed_text['pos']) == len(dependencies)):\n",
        "    count = 0\n",
        "    Question_word_string= \"\"\n",
        "\n",
        "    for pos in parsed_text['pos']:\n",
        "      if(pos==\"WDT\" or pos==\"WRB\" or pos==\"WP\"):\n",
        "        QuestionWord_index = count\n",
        "        Question_word_string = parsed_text['word'][QuestionWord_index]\n",
        "        print(\"test string\" + test_string)\n",
        "        if(Question_word_string.lower() == \"when\" or Question_word_string.lower() == \"where\" or Question_word_string.lower() == \"who\"):\n",
        "          proceed = False\n",
        "\n",
        "        break\n",
        "      count=count+1\n",
        "\n",
        "    question_word_pos_index = 0\n",
        "    window_size =0\n",
        "    count_1 = 0\n",
        "    dependencylist = []\n",
        "    #dependency_list['dependencylist'] = []\n",
        "\n",
        "    #window size logic\n",
        "    flag = False\n",
        "    for i in dependencies:\n",
        "      if(flag):\n",
        "        print(i[1])\n",
        "        dependency_list = i[1]\n",
        "\n",
        "        if(parsed_text['pos'][count_1] != \"NN\" ):\n",
        "          window_size = 5\n",
        "          print(\"window_size\")\n",
        "          print(window_size)\n",
        "        else:\n",
        "          window_size = 3\n",
        "          print(\"window_size\")\n",
        "          print(window_size)\n",
        "        break\n",
        "      if(count_1 == QuestionWord_index):\n",
        "        if(parsed_text['pos'][count_1] == \"WDT\" or parsed_text['pos'][count_1] == \"WRB\" or parsed_text['pos'][count_1] == \"WP\"):\n",
        "          question_word_pos_index = count_1    \n",
        "          flag = True\n",
        "        # write one more condition for corner case where i is in hr   \n",
        "      count_1 = count_1 +1\n",
        "\n",
        "    print(\"window size=\")\n",
        "    print(window_size)\n",
        "\n",
        "    for i in dependencies:\n",
        "      dependencylist.append(i[1])\n",
        "\n",
        "    subject_found = False\n",
        "    boundary = 0\n",
        "\n",
        "    if question_word_pos_index + window_size < len(parsed_text['word']):\n",
        "      boundary = question_word_pos_index + window_size + 1\n",
        "    else:\n",
        "      boundary = len(parsed_text['word'])\n",
        "\n",
        "    first_noun_found = False\n",
        "    #question_word_pos_index +1\n",
        "    LAT_found= False\n",
        "\n",
        "\n",
        "    if(Question_word_string.lower() == \"how\"):\n",
        "      if(QuestionWord_index+1 <= len(parsed_text['word'])-1):\n",
        "        if(parsed_text['pos'][QuestionWord_index+1] == \"JJ\"):\n",
        "          LAT_Index = QuestionWord_index+1  \n",
        "    #Question_word_string\n",
        "    else:\n",
        "      for index in range(question_word_pos_index + 1, boundary):\n",
        "        if(not first_noun_found):\n",
        "          if(parsed_text['pos'][index] == \"NN\" or parsed_text['pos'][index] == \"NNS\" or parsed_text['pos'][index] == \"NNP\"):\n",
        "            LAT_Index = index\n",
        "            first_noun_found = True\n",
        "        if(dependencylist[index] == \"nsubj\" or dependencylist[index] == \"nsubjpass\" or dependencylist[index] == \"nsubj:pass\"):   \n",
        "          subject_found = True\n",
        "          LAT_Index = index\n",
        "          break\n",
        "\n",
        "  if(not proceed):\n",
        "    LAT_String = parsed_text['word'][QuestionWord_index]\n",
        "    print(parsed_text['word'][QuestionWord_index])\n",
        "  else:\n",
        "    LAT_String = parsed_text['word'][LAT_Index]    \n",
        "    print(parsed_text['word'][LAT_Index])\n",
        "  \n",
        "  return LAT_String"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VFhzh9nlwBsd",
        "colab_type": "code",
        "outputId": "ffdb4b2a-2ad0-4d4a-c8cb-de382b45ac36",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 155
        }
      },
      "source": [
        "\n",
        "original_questiontext = \"sai\"\n",
        "LAT = get_LAT(original_questiontext,nlp)\n",
        "print(LAT)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['NNP']\n",
            "['sai']\n",
            "1\n",
            "1\n",
            "window size=\n",
            "0\n",
            "sai\n",
            "sai\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l98vQV4lt8In",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}