word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# word2vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Считаем вектора слов\n",
    "\n",
    "Суммируем все вектора каждого из текстов, сравниваем между собой\n",
    "\n",
    "Суммируем все вектора абзацев, сравниваем послежовательно каждый абзац каждого текста, сравниаем близость по медиане"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.keyedvectors import FastTextKeyedVectors\n",
    "\n",
    "import numpy as np\n",
    "from collections import OrderedDict\n",
    "\n",
    "from scipy.spatial import distance\n",
    "import operator\n",
    "\n",
    "import statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#используем коллекцию предобученных векторов слов\n",
    "fasttext = FastTextKeyedVectors.load(\"araneum_none_fasttextcbow_300_5_2018/araneum_none_fasttextcbow_300_5_2018.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\Anaconda2\\envs\\py36\\lib\\site-packages\\ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ -7.62082413e-02,   1.78887974e-02,  -2.55130790e-02,\n",
       "         2.06156522e-02,  -2.61091962e-02,   1.12581283e-01,\n",
       "        -4.76743840e-02,   4.94216681e-02,  -4.33580689e-02,\n",
       "        -1.23744784e-02,  -8.83364230e-02,  -3.59828062e-02,\n",
       "        -4.24147323e-02,  -5.06249294e-02,   5.97045124e-02,\n",
       "        -2.56801471e-02,  -3.00195757e-02,  -3.64938304e-02,\n",
       "         4.63778824e-02,  -8.25238228e-02,   1.79971680e-02,\n",
       "        -2.25572549e-02,  -8.69684294e-02,   4.85437512e-02,\n",
       "         4.35988493e-02,  -8.77935216e-02,   1.27777234e-01,\n",
       "        -3.75761390e-02,  -8.42803344e-02,  -1.64243355e-02,\n",
       "        -2.13620923e-02,  -6.77805543e-02,  -4.72699478e-02,\n",
       "        -1.73333585e-02,   5.64810522e-02,  -5.13666123e-02,\n",
       "        -6.55516423e-03,  -3.65715362e-02,  -9.69852433e-02,\n",
       "         1.40067218e-02,   1.65425078e-03,   5.25458716e-02,\n",
       "         1.52026797e-02,  -1.95472371e-02,   1.18016839e-01,\n",
       "        -3.33277285e-02,   2.67862696e-02,   2.63341907e-02,\n",
       "         7.68206567e-02,   1.94976442e-02,  -4.24830019e-02,\n",
       "        -6.97431266e-02,  -7.59030730e-02,  -9.64868367e-02,\n",
       "        -1.60852482e-03,  -8.97823423e-02,  -8.20969716e-02,\n",
       "        -6.01850674e-02,   6.81997417e-03,  -9.49739479e-03,\n",
       "        -3.09683196e-02,  -6.14335500e-02,  -1.44407656e-02,\n",
       "        -2.77405605e-02,  -1.26056001e-01,   1.50050953e-04,\n",
       "         5.05202673e-02,   3.13559137e-02,   6.77485466e-02,\n",
       "         1.50138512e-01,  -4.41092029e-02,  -4.56635579e-02,\n",
       "         1.47871738e-02,   1.02615379e-01,  -1.03173137e-01,\n",
       "        -5.01486287e-02,   1.79153327e-02,   1.01886101e-01,\n",
       "         1.09054754e-02,   1.13993682e-01,   6.59506582e-03,\n",
       "         5.08777201e-02,  -4.88355793e-02,   7.33332112e-02,\n",
       "         3.75106372e-02,  -3.11979279e-02,   4.35983092e-02,\n",
       "        -2.24749241e-02,   5.08867875e-02,   2.78288461e-02,\n",
       "        -2.77889054e-03,  -3.27797160e-02,  -3.39106261e-03,\n",
       "        -1.39162131e-02,   1.54537251e-02,  -4.21457402e-02,\n",
       "         2.35536434e-02,   8.04056823e-02,   4.01332416e-02,\n",
       "         6.43964633e-02,   3.03262714e-02,  -9.34736356e-02,\n",
       "        -1.06729746e-01,   6.26985058e-02,  -3.41548547e-02,\n",
       "         1.26987812e-03,  -9.92501378e-02,  -5.08367568e-02,\n",
       "        -2.80845203e-02,  -4.84234095e-02,  -4.86257207e-03,\n",
       "        -9.28510502e-02,   7.02272728e-02,   4.49148566e-02,\n",
       "        -7.12508857e-02,  -9.49139521e-02,   5.40413754e-03,\n",
       "         8.17713439e-02,   1.25525054e-02,  -7.71567225e-02,\n",
       "         6.16520345e-02,  -3.24193239e-02,   9.79479253e-02,\n",
       "        -9.28143710e-02,  -8.35294351e-02,   2.81191822e-02,\n",
       "        -4.55836244e-02,   1.75863728e-02,  -5.15946746e-02,\n",
       "         3.64038907e-02,   6.52060611e-03,  -5.23118600e-02,\n",
       "        -1.58809885e-01,   4.46893871e-02,  -8.55567828e-02,\n",
       "         1.62575729e-02,  -2.57390598e-03,   2.91078314e-02,\n",
       "        -7.56884143e-02,  -8.05023015e-02,   2.05204654e-02,\n",
       "        -1.13676200e-02,  -9.63547751e-02,   9.29680765e-02,\n",
       "         3.49026248e-02,  -9.66022238e-02,   2.11579949e-02,\n",
       "        -8.56449977e-02,   5.54645658e-02,   9.05535929e-03,\n",
       "        -1.36952549e-01,   3.97628285e-02,   8.26978832e-02,\n",
       "         9.92096886e-02,  -1.07388664e-02,   1.12667400e-02,\n",
       "        -3.59976590e-02,  -1.12371087e-01,   9.34447795e-02,\n",
       "         4.53702137e-02,   3.04346662e-02,   9.76879895e-03,\n",
       "        -4.79200818e-02,  -9.18326676e-02,  -4.29482199e-03,\n",
       "         5.37907705e-02,   6.09475598e-02,  -5.32703511e-02,\n",
       "        -8.27628970e-02,  -1.53610101e-02,  -5.77416122e-02,\n",
       "         3.80002744e-02,  -1.69023257e-02,   4.89608534e-02,\n",
       "         1.54108452e-02,   1.05558857e-01,  -8.57717246e-02,\n",
       "        -7.39997029e-02,   5.93373440e-02,  -4.38327901e-02,\n",
       "         9.54775363e-02,   4.66787294e-02,  -8.85734111e-02,\n",
       "        -6.40937015e-02,  -6.94125146e-03,  -2.04972606e-02,\n",
       "        -7.52232447e-02,   4.12907749e-02,   3.34083401e-02,\n",
       "         6.47940487e-02,   2.61339229e-02,  -3.73880863e-02,\n",
       "        -1.18172944e-01,  -6.95823431e-02,  -1.19830491e-02,\n",
       "         5.81505336e-02,   7.29095712e-02,   8.21881741e-02,\n",
       "         1.37898317e-02,   1.02520566e-02,  -5.90978377e-02,\n",
       "         4.68842722e-02,   7.89525434e-02,  -9.43082944e-03,\n",
       "         4.13195305e-02,   1.45249069e-02,   5.21720201e-02,\n",
       "         1.42954476e-02,   3.04558761e-02,  -8.27642381e-02,\n",
       "         3.54703516e-02,  -1.22996455e-03,  -7.77624734e-03,\n",
       "        -2.07285751e-02,  -3.10476534e-02,   1.28533747e-02,\n",
       "        -5.60618080e-02,  -1.63129985e-03,  -8.43731537e-02,\n",
       "        -8.62453058e-02,   2.56215129e-03,   4.32207808e-02,\n",
       "        -7.90902674e-02,  -3.31873377e-03,  -3.17806122e-03,\n",
       "         2.08760649e-02,  -1.54949687e-02,  -6.44365847e-02,\n",
       "        -1.24313021e-02,  -8.11609626e-02,   2.22701207e-02,\n",
       "         4.69396217e-03,   5.89911118e-02,   1.03862612e-02,\n",
       "         1.36910575e-02,   1.20451627e-02,   4.65143425e-03,\n",
       "        -1.15861535e-01,   6.43518344e-02,   7.23849088e-02,\n",
       "        -5.16980328e-03,   3.31692472e-02,   9.76368878e-03,\n",
       "         2.40363590e-02,  -6.33898675e-02,   7.77738094e-02,\n",
       "        -6.66663051e-02,  -1.49262631e-02,   6.68593124e-02,\n",
       "         2.86839884e-02,  -2.61917971e-02,  -1.10093188e-02,\n",
       "         1.64363496e-02,   4.02141456e-03,   7.52408523e-03,\n",
       "         6.24245927e-02,   3.75544876e-02,   2.24200469e-02,\n",
       "        -1.06782004e-01,   6.43170327e-02,   1.04747089e-02,\n",
       "        -7.15828016e-02,   9.94295552e-02,   2.98374649e-02,\n",
       "         7.85854366e-03,   7.94009045e-02,   2.37248503e-02,\n",
       "        -1.56071372e-02,   2.52206195e-02,   4.98638637e-02,\n",
       "         3.80529612e-02,  -5.69499843e-02,  -9.07995459e-03,\n",
       "         4.31180075e-02,   8.01347420e-02,   6.60736561e-02,\n",
       "         4.44461592e-02,  -5.50953634e-02,   2.44012894e-03,\n",
       "        -2.62258612e-02,   6.79651201e-02,   5.95651194e-02,\n",
       "         4.30838019e-02,  -1.42117338e-02,  -1.97396018e-02,\n",
       "         5.25664985e-02,   5.10240858e-03,  -4.69278451e-03,\n",
       "        -9.76828709e-02,   3.05485651e-02,   1.15614915e-02,\n",
       "         6.97491178e-03,  -1.99802616e-03,   8.77728313e-02,\n",
       "         1.22595854e-01,   1.16273925e-01,   3.81238610e-02,\n",
       "         9.38116666e-03,  -9.00964439e-02,  -1.80277806e-02], dtype=float32)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#проверяем что все корректно подгрузилось\n",
    "fasttext['уважаемый'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_and_vectorise_file(file):\n",
    "    \"\"\"переводим файл в набор листов, каждый элемент которых соответствует вектору слова\"\"\"\n",
    "    tokenized_text = []\n",
    "    words_not_found = 0\n",
    "    with open (file, \"r\", encoding = \"utf-8\") as file:\n",
    "        for line in file:\n",
    "            tokenized_line = line.split(' ')\n",
    "            tokenized_line = [tok.replace('\\n', '')  for tok in tokenized_line if (len(tok) > 2)]\n",
    "            if(tokenized_line):\n",
    "                vectors_list = []\n",
    "                for word in tokenized_line:\n",
    "                    try:\n",
    "                        vector = fasttext[word]\n",
    "                        vectors_list.append(vector)\n",
    "                    except:\n",
    "                        words_not_found += 1\n",
    "                if(vectors_list):\n",
    "                    tokenized_text.append(vectors_list)\n",
    "    print(\"words_not_found\", words_not_found)\n",
    "    return tokenized_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Asus\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:12: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  if sys.path[0] == '':\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "words_not_found 29\n",
      "words_not_found 52\n",
      "words_not_found 2\n",
      "words_not_found 46\n",
      "words_not_found 22\n"
     ]
    }
   ],
   "source": [
    "putin = tokenize_and_vectorise_file('speech_putin_processed.txt')\n",
    "rzn = tokenize_and_vectorise_file('speech_rzn_processed.txt')\n",
    "tmn = tokenize_and_vectorise_file('speech_tmn_processed.txt')\n",
    "tula = tokenize_and_vectorise_file('speech_tula_processed.txt')\n",
    "yml = tokenize_and_vectorise_file('speech_yml_processed.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sum_text_vectores_by_word(text):\n",
    "    \"\"\"суммируем все вектора слов текста\"\"\"\n",
    "    vector = np.zeros(len(text[0][0]))\n",
    "    for line in text:\n",
    "        for word_vec in line:\n",
    "            vector = np.add(vector,word_vec)\n",
    "    return vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "putin_vector = sum_text_vectores_by_word(putin)\n",
    "rzn_vector = sum_text_vectores_by_word(rzn)\n",
    "tmn_vector = sum_text_vectores_by_word(tmn)\n",
    "tula_vector = sum_text_vectores_by_word(tula)\n",
    "yml_vector = sum_text_vectores_by_word(yml)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Попарно без повторений находим косинусную близость каждого из полученных векторов. Полчаем выводы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ryazan vs tyumen\n",
      "0.07299630528692735\n",
      "ryazan vs tula\n",
      "0.01810150656670917\n",
      "ryazan vs yamal\n",
      "0.025649192890273675\n",
      "ryazan vs putin\n",
      "0.10347123788346713\n",
      "\n",
      "tyumen vs tula\n",
      "0.06503363904335835\n",
      "tyumen vs yamal\n",
      "0.0758213082958028\n",
      "tyumen vs putin\n",
      "0.031180624158894776\n",
      "\n",
      "tula vs yamal\n",
      "0.02292316631166158\n",
      "tula vs putin\n",
      "0.08446735731500388\n",
      "\n",
      "yamal vs putin\n",
      "0.0914573075919699\n",
      "\n",
      "\n",
      "ryazan is most similar to Putin speech\n"
     ]
    }
   ],
   "source": [
    "vectors_dict = {'ryazan':rzn_vector,\"tyumen\":tmn_vector,\"tula\": tula_vector, \"yamal\": yml_vector, \"putin\":putin_vector}\n",
    "putin_cosine_distance_dict = {'ryazan':0,\"tyumen\":0,\"tula\": 0, \"yamal\": 0}\n",
    "ordered_vectors_dict = OrderedDict(vectors_dict)\n",
    "keys = list(ordered_vectors_dict.keys())\n",
    "values = list(ordered_vectors_dict.values())\n",
    "\n",
    "for dict_ind in range(len(keys)):\n",
    "    current_vect = values[dict_ind]\n",
    "    for compare_ind in range(dict_ind + 1,len(keys)):\n",
    "        print(keys[dict_ind], \"vs\", keys[compare_ind])\n",
    "        compared_vect = values[compare_ind]\n",
    "        print(distance.cosine(current_vect,compared_vect))\n",
    "        \n",
    "        if(keys[compare_ind] == 'putin'):\n",
    "            putin_cosine_distance_dict[keys[dict_ind]] = distance.cosine(current_vect,compared_vect)\n",
    "    print()\n",
    "sorted_x = sorted(putin_cosine_distance_dict.items(), key=operator.itemgetter(1))\n",
    "print(sorted_x[-1][0],\"is most similar to Putin speech\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sum_text_vectores_by_parahraph(text):\n",
    "    \"\"\"суммируем векторы слов внутри каждого абзаца\"\"\"\n",
    "    paragraph_vectors = []\n",
    "    for line in text:\n",
    "        paragraph_vector = np.zeros(len(text[0][0]))\n",
    "        for word_vec in line:\n",
    "            paragraph_vector = np.add(paragraph_vector,word_vec)\n",
    "        paragraph_vectors.append(paragraph_vector)\n",
    "    return paragraph_vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "putin_par_vectors = sum_text_vectores_by_parahraph(putin)\n",
    "rzn_par_vectors = sum_text_vectores_by_parahraph(rzn)\n",
    "tmn_par_vectors = sum_text_vectores_by_parahraph(tmn)\n",
    "tula_par_vectors = sum_text_vectores_by_parahraph(tula)\n",
    "yml_par_vectors = sum_text_vectores_by_parahraph(yml)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_text_vectors_by_paragraphs(text_vect_1, text_vect_2):\n",
    "    \"\"\"по очереди сравниваем все абзацы каждого из текстов, возвращаем медиану косинусных близостей\"\"\"\n",
    "    dist_list = []\n",
    "    for first_ind in range(len(text_vect_1)):\n",
    "        for second_ind in range(first_ind + 1, len(text_vect_2)):\n",
    "            dist = distance.cosine(text_vect_1[first_ind],text_vect_2[second_ind])\n",
    "            dist_list.append(dist)\n",
    "    dist_list_median = statistics.median(dist_list)\n",
    "    return dist_list_median"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Попарно без повторений находим медиану косинусных близостей каждого из полученных векторов. Получаем выводы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ryazan vs tyumen\n",
      "0.5728976662871587\n",
      "ryazan vs tula\n",
      "0.4949619878988849\n",
      "ryazan vs yamal\n",
      "0.48090028602298207\n",
      "ryazan vs putin\n",
      "0.5456994849176036\n",
      "\n",
      "tyumen vs tula\n",
      "0.5704417922255496\n",
      "tyumen vs yamal\n",
      "0.5334847802975864\n",
      "tyumen vs putin\n",
      "0.5510098663350491\n",
      "\n",
      "tula vs yamal\n",
      "0.48787662434812407\n",
      "tula vs putin\n",
      "0.5508491821354387\n",
      "\n",
      "yamal vs putin\n",
      "0.5422966409752872\n",
      "\n",
      "\n",
      "tyumen is most similar to Putin speech\n"
     ]
    }
   ],
   "source": [
    "par_vectors_dict = {'ryazan':rzn_par_vectors,\"tyumen\":tmn_par_vectors,\"tula\": tula_par_vectors, \"yamal\": yml_par_vectors, \"putin\":putin_par_vectors}\n",
    "putin_cosine_distance_dict = {'ryazan':0,\"tyumen\":0,\"tula\": 0, \"yamal\": 0}\n",
    "ordered_vectors_dict = OrderedDict(par_vectors_dict)\n",
    "keys = list(ordered_vectors_dict.keys())\n",
    "values = list(ordered_vectors_dict.values())\n",
    "\n",
    "for dict_ind in range(len(keys)):\n",
    "    current_vect = values[dict_ind]\n",
    "    for compare_ind in range(dict_ind + 1,len(keys)):\n",
    "        print(keys[dict_ind], \"vs\", keys[compare_ind])\n",
    "        compared_vect = values[compare_ind]\n",
    "        print(compare_text_vectors_by_paragraphs(current_vect,compared_vect))\n",
    "        \n",
    "        if(keys[compare_ind] == 'putin'):\n",
    "            putin_cosine_distance_dict[keys[dict_ind]] = compare_text_vectors_by_paragraphs(current_vect,compared_vect)\n",
    "    print()\n",
    "sorted_x = sorted(putin_cosine_distance_dict.items(), key=operator.itemgetter(1))\n",
    "print(sorted_x[-1][0],\"is most similar to Putin speech\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}