Part_of_Speech counter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Анализ частотных существительных, прилагательных и глаголов"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Считаем частотность всех указанных частей речи в каждом\n",
    "\n",
    "Выводим топ 10 частотных слов на каждую часть речи\n",
    "\n",
    "Анализируем общие слова из выведенных топ 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pymystem3 import Mystem\n",
    "import string\n",
    "\n",
    "import operator\n",
    "\n",
    "import progressbar\n",
    "import time\n",
    "\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_dict(word, pos_dict):\n",
    "    \"\"\"инкрементируем уже существующий элемент словаря или создаем несуществующий элемент со значением 1\"\"\"\n",
    "    if(word not in pos_dict):\n",
    "        pos_dict[word] = 1\n",
    "    else:\n",
    "        pos_dict[word] += 1\n",
    "        \n",
    "def verb_noun_adj_dict_creator(mystemmed_line, noun_dict, adj_dict, verb_dict):\n",
    "    \"\"\" создаем словарь частотности глаголов, существительных и прилагательных \"\"\"\n",
    "    for mystemmed_word in mystemmed_line:\n",
    "        keys = list(mystemmed_word.keys())\n",
    "        values = list(mystemmed_word.values())\n",
    "        word  = mystemmed_word['text']\n",
    "        if ('analysis' not in keys):\n",
    "            pass\n",
    "        elif(mystemmed_word['analysis'] ==[]):\n",
    "            pass\n",
    "        else:\n",
    "            grammar = mystemmed_word['analysis'][0]['gr']\n",
    "            grammar_sep_by_comma = grammar.split(',')\n",
    "            if (len(grammar_sep_by_comma) == 1):\n",
    "                pass\n",
    "            else:\n",
    "                pos = re.match('[A-Z\\s]+', grammar_sep_by_comma[0])[0]\n",
    "                if(pos == \"S\"):                    \n",
    "                    update_dict(values[-1], noun_dict)\n",
    "                elif(pos == \"A\"):\n",
    "                    update_dict(values[-1], adj_dict)\n",
    "                elif(pos == \"V\"):\n",
    "                    update_dict(values[-1], verb_dict)\n",
    "\n",
    "def get_top_pos(ready_pos_dict,top_n):\n",
    "    \"\"\"сортируем словарь частотностей возвращаем top_n самых частотных слов\"\"\"\n",
    "    top_pos_list = []\n",
    "    sorted_dict_items = sorted(ready_pos_dict.items(), key=lambda kv: kv[1],reverse=True)\n",
    "    top_words_count = min(len(ready_pos_dict), top_n)\n",
    "    for word_index in range(top_words_count):\n",
    "        top_pos_list.append(sorted_dict_items[word_index][0])\n",
    "    return top_pos_list\n",
    "                    \n",
    "def verb_noun_adj_dict_creator_file(file):\n",
    "    noun_dict = {}\n",
    "    adj_dict ={}\n",
    "    verb_dict = {}\n",
    "    m = Mystem()\n",
    "    line_number = 0\n",
    "    with open (file, \"r\", encoding = \"utf-8\") as file:\n",
    "        all_lines = file.readlines()\n",
    "        bar = progressbar.ProgressBar(maxval=len(all_lines),\n",
    "                                      widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])\n",
    "        bar.start()\n",
    "        for line in all_lines:\n",
    "            if(line != '\\n' ):\n",
    "                mystemmed_line = m.analyze(line)\n",
    "                verb_noun_adj_dict_creator( mystemmed_line, noun_dict, adj_dict, verb_dict)\n",
    "            line_number += 1\n",
    "            bar.update(line_number)\n",
    "            time.sleep(0.1)\n",
    "    top_nouns = get_top_pos(noun_dict, 10)\n",
    "    top_adj = get_top_pos(adj_dict, 10)\n",
    "    top_verb = get_top_pos(verb_dict, 10)\n",
    "    return top_nouns, top_adj, top_verb\n",
    "\n",
    "#top_nouns_rzn, top_adj_rzn, top_verbs_rzn = verb_noun_adj_dict_creator_file('speech_rzn_processed.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[======================================================================= ]  99%\r"
     ]
    }
   ],
   "source": [
    "top_nouns_rzn,top_adj_rzn,top_verbs_rzn= verb_noun_adj_dict_creator_file('speech_rzn_processed.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[======================================================================= ]  99%\r"
     ]
    }
   ],
   "source": [
    "top_nouns_tmn, top_adj_tmn, top_verbs_tmn = verb_noun_adj_dict_creator_file('speech_tmn_processed.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[======================================================================= ]  99%\r"
     ]
    }
   ],
   "source": [
    "top_nouns_tula, top_adj_tula, top_verbs_tula = verb_noun_adj_dict_creator_file('speech_tula_processed.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[======================================================================= ]  99%\r"
     ]
    }
   ],
   "source": [
    "top_nouns_yml, top_adj_yml, top_verbs_yml = verb_noun_adj_dict_creator_file('speech_yml_processed.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[======================================================================= ]  99%\r"
     ]
    }
   ],
   "source": [
    "top_nouns_putin, top_adj_putin, top_verbs_putin = verb_noun_adj_dict_creator_file('speech_putin_processed.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_pos_words = {'ryazan':[top_nouns_rzn,top_adj_rzn,top_verbs_rzn],\n",
    "             \"tyumen\":[top_nouns_tmn, top_adj_tmn, top_verbs_tmn ],\n",
    "             \"tula\": [top_nouns_tula, top_adj_tula, top_verbs_tula], \n",
    "             \"yamal\": [top_nouns_yml, top_adj_yml, top_verbs_yml], \n",
    "             \"putin\":[top_nouns_putin, top_adj_putin, top_verbs_putin]}\n",
    "\n",
    "putin_similarity_dict = {'ryazan':0,\"tyumen\":0,\"tula\": 0, \"yamal\": 0}\n",
    "top_pos_words_ordered_dict = OrderedDict(top_pos_words)\n",
    "keys = list(top_pos_words_ordered_dict.keys())\n",
    "\n",
    "values = list(top_pos_words_ordered_dict.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lists_intersection(lst1, lst2): \n",
    "    \n",
    "    intersection = [value for value in lst1 if value in lst2] \n",
    "    return  intersection \n",
    "\n",
    "def verb_adj_nouns_intersection(first_top_w_list, second_top_w_list, putin):\n",
    "    \"\"\"сравниваем самые частотные части речи двух текстов, если сравниваем с речью Путина, \n",
    "    то возвращаем количесвто общих слов для далнейшего подсчет\"\"\"\n",
    "    intersections = []\n",
    "    common_words_count = 0\n",
    "    for pos_top_words_index in range(3):\n",
    "        pos_intersection = lists_intersection (first_top_w_list[pos_top_words_index], second_top_w_list[pos_top_words_index]) \n",
    "        intersections.append(pos_intersection)\n",
    "    intersections_dict = {'nouns':intersections[0] ,'adjectives' : intersections[1], 'verbs' : intersections[2]}\n",
    "    for pos, pos_intersection_list in intersections_dict.items():\n",
    "        if(pos_intersection_list):\n",
    "            common_words_count += len(pos_intersection_list)\n",
    "            print(\"Common\", pos, \":\", pos_intersection_list)\n",
    "        else:\n",
    "            print(\"No common\", pos)\n",
    "    if (putin):\n",
    "        return common_words_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ryazan vs tyumen\n",
      "Common nouns : ['год', 'область', 'проект', 'человек', 'развитие']\n",
      "Common adjectives : ['новый', 'должный', 'социальный', 'общественный', 'хороший']\n",
      "Common verbs : ['сделать', 'становиться', 'обеспечивать', 'продолжать', 'создавать']\n",
      "ryazan vs tula\n",
      "Common nouns : ['год', 'регион', 'область', 'проект', 'развитие', 'работа']\n",
      "Common adjectives : ['новый', 'должный', 'социальный', 'федеральный']\n",
      "Common verbs : ['становиться', 'создавать']\n",
      "ryazan vs yamal\n",
      "Common nouns : ['год', 'регион', 'проект', 'развитие', 'работа']\n",
      "Common adjectives : ['новый', 'социальный', 'федеральный']\n",
      "Common verbs : ['становиться', 'продолжать']\n",
      "ryazan vs putin\n",
      "Common nouns : ['год', 'человек', 'развитие']\n",
      "Common adjectives : ['новый', 'должный', 'современный']\n",
      "Common verbs : ['позволять', 'сделать', 'становиться', 'обеспечивать', 'создавать']\n",
      "\n",
      "tyumen vs tula\n",
      "Common nouns : ['область', 'год', 'проект', 'развитие', 'гражданин']\n",
      "Common adjectives : ['новый', 'должный', 'социальный', 'медицинский', 'региональный']\n",
      "Common verbs : ['становиться', 'создавать', 'давать']\n",
      "tyumen vs yamal\n",
      "Common nouns : ['год', 'проект', 'развитие', 'россия']\n",
      "Common adjectives : ['новый', 'социальный']\n",
      "Common verbs : ['становиться', 'продолжать', 'оставаться']\n",
      "tyumen vs putin\n",
      "Common nouns : ['год', 'развитие', 'человек', 'россия']\n",
      "Common adjectives : ['новый', 'должный']\n",
      "Common verbs : ['становиться', 'сделать', 'говорить', 'создавать', 'работать', 'обеспечивать']\n",
      "\n",
      "tula vs yamal\n",
      "Common nouns : ['год', 'работа', 'проект', 'развитие', 'регион']\n",
      "Common adjectives : ['новый', 'социальный', 'федеральный', 'большой']\n",
      "Common verbs : ['становиться', 'принимать']\n",
      "tula vs putin\n",
      "Common nouns : ['год', 'развитие']\n",
      "Common adjectives : ['новый', 'должный', 'большой']\n",
      "Common verbs : ['становиться', 'создавать']\n",
      "\n",
      "yamal vs putin\n",
      "Common nouns : ['год', 'развитие', 'россия']\n",
      "Common adjectives : ['новый', 'уважаемый', 'большой']\n",
      "Common verbs : ['становиться']\n",
      "\n",
      "\n",
      "tyumen is most similar to Putin speech\n"
     ]
    }
   ],
   "source": [
    "for dict_ind in range(len(keys)):\n",
    "    current_top_w_list = values[dict_ind]\n",
    "    for compare_ind in range(dict_ind + 1,len(keys)):\n",
    "        print(keys[dict_ind], \"vs\", keys[compare_ind])\n",
    "        compared_top_w_list = values[compare_ind]\n",
    "        \n",
    "        if(keys[compare_ind] != 'putin'):\n",
    "            verb_adj_nouns_intersection(current_top_w_list, compared_top_w_list, putin = False)\n",
    "\n",
    "\n",
    "        elif(keys[compare_ind] == 'putin'):\n",
    "            common_words = verb_adj_nouns_intersection(current_top_w_list, compared_top_w_list, putin = True)\n",
    "            putin_similarity_dict[keys[dict_ind]] = common_words\n",
    "\n",
    "    print()\n",
    "sorted_x = sorted(putin_similarity_dict.items(), key=operator.itemgetter(1))\n",
    "print(sorted_x[-1][0],\"is most similar to Putin speech\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}